2024-09-17T00:46:30.954 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-17T00:46:30.972 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T00:46:31.047 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7907988' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: e20bff7d16f5b09c58ead9548e8cf98cf87ee227 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 599 sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f sleep_before_teardown: 0 subset: 33/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: e20bff7d16f5b09c58ead9548e8cf98cf87ee227 targets: smithi033.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHdDvxD54rzsbUA1HCJGEB9TTJEPNGHJZ+c+zz5lTYyTMXtjwWV1X11u88fDBnhuqshy1HPtRc/lKmP62kp1NQ4= smithi100.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMKbqnZUXjlsjDPsDbx9jFAQYeHnflLK2QXsCGU2j+PNBzNfhUi1HKRZha636Q7qkMggRC9jWPGZhnyFrJIDJKY= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_21:08:04 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-17T00:46:31.047 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa; will attempt to use it 2024-09-17T00:46:31.048 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa/tasks 2024-09-17T00:46:31.048 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-17T00:46:31.050 INFO:teuthology.task.internal:Checking packages... 2024-09-17T00:46:31.070 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c2ddf9721ef254645201cdbabd9c8f49c30a586f' 2024-09-17T00:46:31.070 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-17T00:46:31.070 INFO:teuthology.packaging:ref: None 2024-09-17T00:46:31.070 INFO:teuthology.packaging:tag: None 2024-09-17T00:46:31.071 INFO:teuthology.packaging:branch: squid 2024-09-17T00:46:31.071 INFO:teuthology.packaging:sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T00:46:31.071 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-17T00:46:31.286 INFO:teuthology.task.internal:Found packages for ceph version 19.2.0-0 2024-09-17T00:46:31.288 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-17T00:46:31.296 INFO:teuthology.task.internal:no buildpackages task found 2024-09-17T00:46:31.296 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-17T00:46:31.358 INFO:teuthology.task.internal:Saving configuration 2024-09-17T00:46:31.371 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-17T00:46:31.414 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-17T00:46:31.439 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi033.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 00:41:42.452654', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHdDvxD54rzsbUA1HCJGEB9TTJEPNGHJZ+c+zz5lTYyTMXtjwWV1X11u88fDBnhuqshy1HPtRc/lKmP62kp1NQ4='} 2024-09-17T00:46:31.463 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi100.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-17 00:41:42.451254', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMKbqnZUXjlsjDPsDbx9jFAQYeHnflLK2QXsCGU2j+PNBzNfhUi1HKRZha636Q7qkMggRC9jWPGZhnyFrJIDJKY='} 2024-09-17T00:46:31.463 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-17T00:46:31.481 INFO:teuthology.task.internal:roles: ubuntu@smithi033.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-17T00:46:31.481 INFO:teuthology.task.internal:roles: ubuntu@smithi100.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-17T00:46:31.481 INFO:teuthology.run_tasks:Running task console_log... 2024-09-17T00:46:31.574 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f2afc8bbac0>, signals=[15]) 2024-09-17T00:46:31.574 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-17T00:46:31.579 INFO:teuthology.task.internal:Opening connections... 2024-09-17T00:46:31.580 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T00:46:31.581 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:46:31.661 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi100.front.sepia.ceph.com 2024-09-17T00:46:31.661 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:46:31.740 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-17T00:46:31.748 DEBUG:teuthology.orchestra.run.smithi033:> uname -m 2024-09-17T00:46:31.766 INFO:teuthology.orchestra.run.smithi033.stdout:x86_64 2024-09-17T00:46:31.767 DEBUG:teuthology.orchestra.run.smithi033:> cat /etc/os-release 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:NAME="CentOS Stream" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION="9" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:ID="centos" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:ID_LIKE="rhel fedora" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION_ID="9" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:PLATFORM_ID="platform:el9" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T00:46:31.825 INFO:teuthology.orchestra.run.smithi033.stdout:ANSI_COLOR="0;31" 2024-09-17T00:46:31.826 INFO:teuthology.orchestra.run.smithi033.stdout:LOGO="fedora-logo-icon" 2024-09-17T00:46:31.826 INFO:teuthology.orchestra.run.smithi033.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T00:46:31.826 INFO:teuthology.orchestra.run.smithi033.stdout:HOME_URL="https://centos.org/" 2024-09-17T00:46:31.826 INFO:teuthology.orchestra.run.smithi033.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T00:46:31.826 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T00:46:31.826 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T00:46:31.827 INFO:teuthology.lock.ops:Updating smithi033.front.sepia.ceph.com on lock server 2024-09-17T00:46:31.849 DEBUG:teuthology.orchestra.run.smithi100:> uname -m 2024-09-17T00:46:31.867 INFO:teuthology.orchestra.run.smithi100.stdout:x86_64 2024-09-17T00:46:31.867 DEBUG:teuthology.orchestra.run.smithi100:> cat /etc/os-release 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:NAME="CentOS Stream" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:VERSION="9" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:ID="centos" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:ID_LIKE="rhel fedora" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:VERSION_ID="9" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:PLATFORM_ID="platform:el9" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-17T00:46:31.926 INFO:teuthology.orchestra.run.smithi100.stdout:ANSI_COLOR="0;31" 2024-09-17T00:46:31.927 INFO:teuthology.orchestra.run.smithi100.stdout:LOGO="fedora-logo-icon" 2024-09-17T00:46:31.927 INFO:teuthology.orchestra.run.smithi100.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-17T00:46:31.927 INFO:teuthology.orchestra.run.smithi100.stdout:HOME_URL="https://centos.org/" 2024-09-17T00:46:31.927 INFO:teuthology.orchestra.run.smithi100.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-17T00:46:31.927 INFO:teuthology.orchestra.run.smithi100.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-17T00:46:31.927 INFO:teuthology.orchestra.run.smithi100.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-17T00:46:31.927 INFO:teuthology.lock.ops:Updating smithi100.front.sepia.ceph.com on lock server 2024-09-17T00:46:31.949 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-17T00:46:31.957 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-17T00:46:31.995 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-17T00:46:31.995 DEBUG:teuthology.orchestra.run.smithi033:> test '!' -e /home/ubuntu/cephtest 2024-09-17T00:46:31.998 DEBUG:teuthology.orchestra.run.smithi100:> test '!' -e /home/ubuntu/cephtest 2024-09-17T00:46:32.015 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-17T00:46:32.044 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-17T00:46:32.044 DEBUG:teuthology.orchestra.run.smithi033:> test -z $(ls -A /var/lib/ceph) 2024-09-17T00:46:32.055 DEBUG:teuthology.orchestra.run.smithi100:> test -z $(ls -A /var/lib/ceph) 2024-09-17T00:46:32.147 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-17T00:46:32.199 INFO:teuthology.run_tasks:Running task kernel... 2024-09-17T00:46:32.215 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-17T00:46:32.215 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-17T00:46:32.215 DEBUG:teuthology.orchestra.run.smithi033:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T00:46:32.216 DEBUG:teuthology.orchestra.run.smithi100:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-17T00:46:32.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:46:32.234 DEBUG:teuthology.orchestra.run.smithi033:> uname -r 2024-09-17T00:46:32.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:46:32.235 DEBUG:teuthology.orchestra.run.smithi100:> uname -r 2024-09-17T00:46:32.291 INFO:teuthology.orchestra.run.smithi033.stdout:5.14.0-503.el9.x86_64 2024-09-17T00:46:32.292 INFO:teuthology.task.kernel:Running kernel on smithi033: 5.14.0-503.el9.x86_64 2024-09-17T00:46:32.292 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum install -y kernel 2024-09-17T00:46:32.293 INFO:teuthology.orchestra.run.smithi100.stdout:5.14.0-503.el9.x86_64 2024-09-17T00:46:32.293 INFO:teuthology.task.kernel:Running kernel on smithi100: 5.14.0-503.el9.x86_64 2024-09-17T00:46:32.293 DEBUG:teuthology.orchestra.run.smithi100:> sudo yum install -y kernel 2024-09-17T00:46:35.425 INFO:teuthology.orchestra.run.smithi100.stdout:CentOS Stream 9 - BaseOS 6.0 MB/s | 8.3 MB 00:01 2024-09-17T00:46:36.027 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - BaseOS 4.4 MB/s | 8.3 MB 00:01 2024-09-17T00:46:44.138 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - AppStream 2.9 MB/s | 20 MB 00:06 2024-09-17T00:46:44.291 INFO:teuthology.orchestra.run.smithi100.stdout:CentOS Stream 9 - AppStream 2.6 MB/s | 20 MB 00:07 2024-09-17T00:46:48.950 INFO:teuthology.orchestra.run.smithi100.stdout:CentOS Stream 9 - CRB 7.7 MB/s | 6.5 MB 00:00 2024-09-17T00:46:49.944 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - CRB 3.5 MB/s | 6.5 MB 00:01 2024-09-17T00:46:50.534 INFO:teuthology.orchestra.run.smithi100.stdout:CentOS Stream 9 - Extras packages 71 kB/s | 19 kB 00:00 2024-09-17T00:46:51.503 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - Extras packages 65 kB/s | 19 kB 00:00 2024-09-17T00:46:52.066 INFO:teuthology.orchestra.run.smithi100.stdout:Extra Packages for Enterprise Linux 16 MB/s | 23 MB 00:01 2024-09-17T00:46:52.918 INFO:teuthology.orchestra.run.smithi033.stdout:Extra Packages for Enterprise Linux 18 MB/s | 23 MB 00:01 2024-09-17T00:46:57.845 INFO:teuthology.orchestra.run.smithi100.stdout:lab-extras 4.5 kB/s | 1.7 kB 00:00 2024-09-17T00:46:58.830 INFO:teuthology.orchestra.run.smithi033.stdout:lab-extras 5.2 kB/s | 1.7 kB 00:00 2024-09-17T00:46:59.684 INFO:teuthology.orchestra.run.smithi100.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T00:46:59.684 INFO:teuthology.orchestra.run.smithi100.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T00:46:59.780 INFO:teuthology.orchestra.run.smithi100.stdout:Dependencies resolved. 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout:================================================================================ 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout: Package Architecture Version Repository Size 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout:================================================================================ 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout:Installing: 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout:Installing dependencies: 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout:Transaction Summary 2024-09-17T00:46:59.787 INFO:teuthology.orchestra.run.smithi100.stdout:================================================================================ 2024-09-17T00:46:59.788 INFO:teuthology.orchestra.run.smithi100.stdout:Install 4 Packages 2024-09-17T00:46:59.788 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:46:59.788 INFO:teuthology.orchestra.run.smithi100.stdout:Total download size: 79 M 2024-09-17T00:46:59.788 INFO:teuthology.orchestra.run.smithi100.stdout:Installed size: 126 M 2024-09-17T00:46:59.788 INFO:teuthology.orchestra.run.smithi100.stdout:Downloading Packages: 2024-09-17T00:46:59.866 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-5.14.0-508.el9.x86_64.rpm: Status code: 404 for http://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:46:59.866 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-core-5.14.0-508.el9.x86_64.rpm: Status code: 404 for http://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:46:59.866 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-modules-5.14.0-508.el9.x86_64.rpm: Status code: 404 for http://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:46:59.962 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-5.14.0-508.el9.x86_64.rpm: Status code: 404 for https://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:46:59.967 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-core-5.14.0-508.el9.x86_64.rpm: Status code: 404 for https://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-core-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:46:59.976 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-modules-5.14.0-508.el9.x86_64.rpm: Status code: 404 for https://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:47:00.038 INFO:teuthology.orchestra.run.smithi100.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 157 kB/s | 36 kB 00:00 2024-09-17T00:47:00.066 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-modules-core-5.14.0-508.el9.x86_64.rpm: Status code: 404 for https://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-core-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:47:00.095 INFO:teuthology.orchestra.run.smithi100.stdout:[MIRROR] kernel-modules-core-5.14.0-508.el9.x86_64.rpm: Status code: 404 for http://mirror.team-cymru.com/centos-stream/9-stream/BaseOS/x86_64/os/Packages/kernel-modules-core-5.14.0-508.el9.x86_64.rpm (IP: 216.31.2.234) 2024-09-17T00:47:00.471 INFO:teuthology.orchestra.run.smithi100.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 24 MB/s | 16 MB 00:00 2024-09-17T00:47:00.820 INFO:teuthology.orchestra.run.smithi033.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-17T00:47:00.820 INFO:teuthology.orchestra.run.smithi033.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-17T00:47:00.920 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-09-17T00:47:00.926 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-17T00:47:00.926 INFO:teuthology.orchestra.run.smithi033.stdout: Package Architecture Version Repository Size 2024-09-17T00:47:00.926 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-17T00:47:00.926 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-09-17T00:47:00.926 INFO:teuthology.orchestra.run.smithi033.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout:Install 4 Packages 2024-09-17T00:47:00.927 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:47:00.928 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 79 M 2024-09-17T00:47:00.928 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 126 M 2024-09-17T00:47:00.928 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-09-17T00:47:01.013 INFO:teuthology.orchestra.run.smithi100.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 29 MB/s | 35 MB 00:01 2024-09-17T00:47:01.740 INFO:teuthology.orchestra.run.smithi033.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 64 kB/s | 36 kB 00:00 2024-09-17T00:47:01.805 INFO:teuthology.orchestra.run.smithi100.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 16 MB/s | 29 MB 00:01 2024-09-17T00:47:01.808 INFO:teuthology.orchestra.run.smithi100.stdout:-------------------------------------------------------------------------------- 2024-09-17T00:47:01.808 INFO:teuthology.orchestra.run.smithi100.stdout:Total 39 MB/s | 79 MB 00:02 2024-09-17T00:47:02.488 INFO:teuthology.orchestra.run.smithi100.stdout:Running transaction check 2024-09-17T00:47:02.698 INFO:teuthology.orchestra.run.smithi033.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 10 MB/s | 16 MB 00:01 2024-09-17T00:47:02.839 INFO:teuthology.orchestra.run.smithi100.stdout:Transaction check succeeded. 2024-09-17T00:47:02.839 INFO:teuthology.orchestra.run.smithi100.stdout:Running transaction test 2024-09-17T00:47:03.382 INFO:teuthology.orchestra.run.smithi033.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 16 MB/s | 35 MB 00:02 2024-09-17T00:47:03.782 INFO:teuthology.orchestra.run.smithi100.stdout:Transaction test succeeded. 2024-09-17T00:47:03.782 INFO:teuthology.orchestra.run.smithi100.stdout:Running transaction 2024-09-17T00:47:03.932 INFO:teuthology.orchestra.run.smithi033.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 13 MB/s | 29 MB 00:02 2024-09-17T00:47:03.932 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-09-17T00:47:03.933 INFO:teuthology.orchestra.run.smithi033.stdout:Total 26 MB/s | 79 MB 00:03 2024-09-17T00:47:04.646 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-09-17T00:47:05.022 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-09-17T00:47:05.022 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-09-17T00:47:05.999 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-09-17T00:47:05.999 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-09-17T00:47:06.208 INFO:teuthology.orchestra.run.smithi100.stdout: Preparing : 1/1 2024-09-17T00:47:06.907 INFO:teuthology.orchestra.run.smithi100.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T00:47:06.998 INFO:teuthology.orchestra.run.smithi100.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T00:47:08.579 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-09-17T00:47:09.242 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-17T00:47:09.341 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T00:47:09.747 INFO:teuthology.orchestra.run.smithi100.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T00:47:09.877 INFO:teuthology.orchestra.run.smithi100.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T00:47:12.090 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T00:47:12.229 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T00:47:14.742 INFO:teuthology.orchestra.run.smithi100.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T00:47:14.777 INFO:teuthology.orchestra.run.smithi100.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:17.351 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T00:47:17.379 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:19.614 INFO:teuthology.orchestra.run.smithi100.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:22.524 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:57.902 INFO:teuthology.orchestra.run.smithi100.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:57.941 INFO:teuthology.orchestra.run.smithi100.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:58.312 INFO:teuthology.orchestra.run.smithi100.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:58.312 INFO:teuthology.orchestra.run.smithi100.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T00:47:58.312 INFO:teuthology.orchestra.run.smithi100.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T00:47:58.312 INFO:teuthology.orchestra.run.smithi100.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T00:47:59.124 INFO:teuthology.orchestra.run.smithi100.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:47:59.124 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout:Installed: 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:47:59.125 INFO:teuthology.orchestra.run.smithi100.stdout:Complete! 2024-09-17T00:47:59.474 DEBUG:teuthology.orchestra.run.smithi100:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T00:47:59.609 INFO:teuthology.orchestra.run.smithi100.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T00:47:59.609 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T00:47:59.609 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T00:47:59.610 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T00:47:59.610 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T00:47:59.610 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-17T00:47:59.610 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi100.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T00:47:59.610 DEBUG:teuthology.orchestra.run.smithi100:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T00:47:59.669 INFO:teuthology.orchestra.run.smithi100.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T00:47:59.670 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T00:47:59.670 DEBUG:teuthology.orchestra.run.smithi100:> sudo rpm -qi grub2-tools 2024-09-17T00:47:59.702 INFO:teuthology.orchestra.run.smithi100.stdout:Name : grub2-tools 2024-09-17T00:47:59.702 INFO:teuthology.orchestra.run.smithi100.stdout:Epoch : 1 2024-09-17T00:47:59.702 INFO:teuthology.orchestra.run.smithi100.stdout:Version : 2.06 2024-09-17T00:47:59.702 INFO:teuthology.orchestra.run.smithi100.stdout:Release : 82.el9 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Architecture: x86_64 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Group : Unspecified 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Size : 8274670 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:License : GPLv3+ 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Packager : builder@centos.org 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:Vendor : CentOS 2024-09-17T00:47:59.703 INFO:teuthology.orchestra.run.smithi100.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:Summary : Support tools for GRUB. 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:Description : 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:hardware devices. 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:47:59.704 INFO:teuthology.orchestra.run.smithi100.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T00:47:59.705 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T00:47:59.705 INFO:teuthology.task.kernel:Updating grub on smithi100 to boot 5.14.0-508.el9.x86_64 2024-09-17T00:47:59.705 DEBUG:teuthology.orchestra.run.smithi100:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T00:48:00.391 INFO:teuthology.orchestra.run.smithi100.stderr:Generating grub configuration file ... 2024-09-17T00:48:01.715 INFO:teuthology.orchestra.run.smithi100.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T00:48:01.738 INFO:teuthology.orchestra.run.smithi100.stderr:done 2024-09-17T00:48:01.740 DEBUG:teuthology.orchestra.run.smithi100:> mktemp 2024-09-17T00:48:01.755 INFO:teuthology.orchestra.run.smithi100.stdout:/tmp/tmp.EcwerJTkvi 2024-09-17T00:48:01.755 DEBUG:teuthology.orchestra.run.smithi100:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.EcwerJTkvi 2024-09-17T00:48:01.788 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:48:01.822 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod 0666 /tmp/tmp.EcwerJTkvi 2024-09-17T00:48:01.830 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:48:01.973 DEBUG:teuthology.orchestra.remote:smithi100:/tmp/tmp.EcwerJTkvi is 6KB 2024-09-17T00:48:01.993 DEBUG:teuthology.orchestra.run.smithi100:> rm -fr /tmp/tmp.EcwerJTkvi 2024-09-17T00:48:02.008 DEBUG:teuthology.orchestra.run.smithi100:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T00:48:02.079 INFO:teuthology.orchestra.run.smithi100.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T00:48:02.079 INFO:teuthology.orchestra.run.smithi100.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T00:48:02.079 INFO:teuthology.orchestra.run.smithi100.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T00:48:02.080 INFO:teuthology.orchestra.run.smithi100.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T00:48:02.081 DEBUG:teuthology.orchestra.run.smithi100:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T00:48:02.193 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:48:02.193 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-17T00:48:02.193 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-17T00:48:02.193 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-17T00:48:02.244 DEBUG:teuthology.orchestra.run.smithi100:> sudo shutdown -r now 2024-09-17T00:48:03.065 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-17T00:48:03.065 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:48:03.066 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-09-17T00:48:03.400 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T00:48:03.485 INFO:teuthology.orchestra.run.smithi033.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T00:48:03.486 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T00:48:03.486 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-17T00:48:03.486 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-17T00:48:03.486 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-17T00:48:03.486 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-17T00:48:03.486 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi033.front.sepia.ceph.com, path=None, version=distro) 2024-09-17T00:48:03.486 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q kernel | sort -rV | head -n 1 2024-09-17T00:48:03.544 INFO:teuthology.orchestra.run.smithi033.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-17T00:48:03.545 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-17T00:48:03.545 DEBUG:teuthology.orchestra.run.smithi033:> sudo rpm -qi grub2-tools 2024-09-17T00:48:03.584 INFO:teuthology.orchestra.run.smithi033.stdout:Name : grub2-tools 2024-09-17T00:48:03.584 INFO:teuthology.orchestra.run.smithi033.stdout:Epoch : 1 2024-09-17T00:48:03.584 INFO:teuthology.orchestra.run.smithi033.stdout:Version : 2.06 2024-09-17T00:48:03.585 INFO:teuthology.orchestra.run.smithi033.stdout:Release : 82.el9 2024-09-17T00:48:03.585 INFO:teuthology.orchestra.run.smithi033.stdout:Architecture: x86_64 2024-09-17T00:48:03.585 INFO:teuthology.orchestra.run.smithi033.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-17T00:48:03.585 INFO:teuthology.orchestra.run.smithi033.stdout:Group : Unspecified 2024-09-17T00:48:03.586 INFO:teuthology.orchestra.run.smithi033.stdout:Size : 8274670 2024-09-17T00:48:03.586 INFO:teuthology.orchestra.run.smithi033.stdout:License : GPLv3+ 2024-09-17T00:48:03.586 INFO:teuthology.orchestra.run.smithi033.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-17T00:48:03.586 INFO:teuthology.orchestra.run.smithi033.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-17T00:48:03.586 INFO:teuthology.orchestra.run.smithi033.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-17T00:48:03.587 INFO:teuthology.orchestra.run.smithi033.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-17T00:48:03.587 INFO:teuthology.orchestra.run.smithi033.stdout:Packager : builder@centos.org 2024-09-17T00:48:03.587 INFO:teuthology.orchestra.run.smithi033.stdout:Vendor : CentOS 2024-09-17T00:48:03.587 INFO:teuthology.orchestra.run.smithi033.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-17T00:48:03.587 INFO:teuthology.orchestra.run.smithi033.stdout:Summary : Support tools for GRUB. 2024-09-17T00:48:03.587 INFO:teuthology.orchestra.run.smithi033.stdout:Description : 2024-09-17T00:48:03.588 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:48:03.588 INFO:teuthology.orchestra.run.smithi033.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-17T00:48:03.588 INFO:teuthology.orchestra.run.smithi033.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-17T00:48:03.588 INFO:teuthology.orchestra.run.smithi033.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-17T00:48:03.588 INFO:teuthology.orchestra.run.smithi033.stdout:hardware devices. 2024-09-17T00:48:03.588 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:48:03.589 INFO:teuthology.orchestra.run.smithi033.stdout:This subpackage provides tools for support of all platforms. 2024-09-17T00:48:03.590 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-17T00:48:03.590 INFO:teuthology.task.kernel:Updating grub on smithi033 to boot 5.14.0-508.el9.x86_64 2024-09-17T00:48:03.591 DEBUG:teuthology.orchestra.run.smithi033:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-17T00:48:04.184 INFO:teuthology.orchestra.run.smithi033.stderr:Generating grub configuration file ... 2024-09-17T00:48:05.468 INFO:teuthology.orchestra.run.smithi033.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-17T00:48:05.508 INFO:teuthology.orchestra.run.smithi033.stderr:done 2024-09-17T00:48:05.510 DEBUG:teuthology.orchestra.run.smithi033:> mktemp 2024-09-17T00:48:05.525 INFO:teuthology.orchestra.run.smithi033.stdout:/tmp/tmp.aPTWxL1vaD 2024-09-17T00:48:05.525 DEBUG:teuthology.orchestra.run.smithi033:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.aPTWxL1vaD 2024-09-17T00:48:05.596 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 0666 /tmp/tmp.aPTWxL1vaD 2024-09-17T00:48:05.743 DEBUG:teuthology.orchestra.remote:smithi033:/tmp/tmp.aPTWxL1vaD is 6KB 2024-09-17T00:48:05.791 DEBUG:teuthology.orchestra.run.smithi033:> rm -fr /tmp/tmp.aPTWxL1vaD 2024-09-17T00:48:05.806 DEBUG:teuthology.orchestra.run.smithi033:> sudo /bin/ls /boot/loader/entries || true 2024-09-17T00:48:05.868 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-17T00:48:05.868 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-17T00:48:05.868 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-17T00:48:05.868 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-17T00:48:05.869 DEBUG:teuthology.orchestra.run.smithi033:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-17T00:48:06.007 DEBUG:teuthology.orchestra.run.smithi033:> sudo shutdown -r now 2024-09-17T00:48:32.273 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-17T00:48:32.274 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-09-17T00:48:32.275 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:48:36.013 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-17T00:48:36.014 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-17T00:48:36.014 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:48:50.749 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.100 2024-09-17T00:48:54.333 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.33 2024-09-17T00:48:59.753 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-09-17T00:48:59.753 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:49:02.813 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.100 2024-09-17T00:49:03.335 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-17T00:49:03.335 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:49:14.825 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-09-17T00:49:14.826 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:49:21.726 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.33 2024-09-17T00:49:33.737 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-17T00:49:33.738 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:49:34.038 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-09-17T00:49:34.432 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-17T00:49:34.433 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-17T00:49:34.436 DEBUG:teuthology.orchestra.run.smithi033:> uname -r 2024-09-17T00:49:34.454 INFO:teuthology.orchestra.run.smithi033.stdout:5.14.0-508.el9.x86_64 2024-09-17T00:49:34.455 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T00:49:34.455 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T00:49:34.455 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T00:49:35.455 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-17T00:49:35.456 DEBUG:teuthology.orchestra.run.smithi033:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T00:49:35.525 INFO:teuthology.orchestra.run.smithi033.stdout:ttyS1 2024-09-17T00:49:35.576 DEBUG:teuthology.parallel:result is None 2024-09-17T00:49:46.949 DEBUG:teuthology.orchestra.run.smithi100:> true 2024-09-17T00:49:47.451 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-09-17T00:49:47.451 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-17T00:49:47.452 DEBUG:teuthology.orchestra.run.smithi100:> uname -r 2024-09-17T00:49:47.490 INFO:teuthology.orchestra.run.smithi100.stdout:5.14.0-508.el9.x86_64 2024-09-17T00:49:47.490 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-17T00:49:47.490 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-17T00:49:47.491 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-17T00:49:48.491 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-17T00:49:48.492 DEBUG:teuthology.orchestra.run.smithi100:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-17T00:49:48.571 INFO:teuthology.orchestra.run.smithi100.stdout:ttyS1 2024-09-17T00:49:48.797 DEBUG:teuthology.parallel:result is None 2024-09-17T00:49:48.797 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-17T00:49:48.811 INFO:teuthology.task.internal:Creating test directory... 2024-09-17T00:49:48.811 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T00:49:48.815 DEBUG:teuthology.orchestra.run.smithi100:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-17T00:49:48.833 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-17T00:49:48.868 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-17T00:49:48.927 INFO:teuthology.task.internal:Creating archive directory... 2024-09-17T00:49:48.928 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T00:49:48.930 DEBUG:teuthology.orchestra.run.smithi100:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-17T00:49:48.987 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-17T00:49:48.998 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-17T00:49:48.998 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T00:49:49.012 DEBUG:teuthology.orchestra.run.smithi100:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-17T00:49:49.041 INFO:teuthology.orchestra.run.smithi100.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T00:49:49.043 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T00:49:49.050 INFO:teuthology.orchestra.run.smithi100.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T00:49:49.055 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-17T00:49:49.056 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-17T00:49:49.067 INFO:teuthology.task.internal:Configuring sudo... 2024-09-17T00:49:49.067 DEBUG:teuthology.orchestra.run.smithi033:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T00:49:49.100 DEBUG:teuthology.orchestra.run.smithi100:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-17T00:49:49.126 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-17T00:49:49.136 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-17T00:49:49.136 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T00:49:49.169 DEBUG:teuthology.orchestra.run.smithi100:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-17T00:49:49.188 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T00:49:49.267 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T00:49:49.323 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T00:49:49.406 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T00:49:49.449 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:49:49.449 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T00:49:49.516 DEBUG:teuthology.orchestra.run.smithi100:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T00:49:49.566 DEBUG:teuthology.orchestra.run.smithi100:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-17T00:49:49.603 DEBUG:teuthology.orchestra.run.smithi100:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T00:49:49.683 DEBUG:teuthology.orchestra.run.smithi100:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-17T00:49:49.725 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-09-17T00:49:49.725 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-17T00:49:49.789 DEBUG:teuthology.orchestra.run.smithi033:> sudo service rsyslog restart 2024-09-17T00:49:49.792 DEBUG:teuthology.orchestra.run.smithi100:> sudo service rsyslog restart 2024-09-17T00:49:49.841 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T00:49:49.881 INFO:teuthology.orchestra.run.smithi100.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T00:49:50.381 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-17T00:49:50.391 INFO:teuthology.task.internal:Starting timer... 2024-09-17T00:49:50.391 INFO:teuthology.run_tasks:Running task pcp... 2024-09-17T00:49:50.459 INFO:teuthology.run_tasks:Running task selinux... 2024-09-17T00:49:50.497 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-17T00:49:50.497 DEBUG:teuthology.orchestra.run.smithi033:> sudo service auditd rotate 2024-09-17T00:49:50.591 INFO:teuthology.orchestra.run.smithi033.stdout:Rotating logs: 2024-09-17T00:49:50.593 DEBUG:teuthology.orchestra.run.smithi100:> sudo service auditd rotate 2024-09-17T00:49:50.653 INFO:teuthology.orchestra.run.smithi100.stdout:Rotating logs: 2024-09-17T00:49:50.655 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-17T00:49:50.655 DEBUG:teuthology.orchestra.run.smithi033:> /usr/sbin/getenforce 2024-09-17T00:49:50.686 INFO:teuthology.orchestra.run.smithi033.stdout:Permissive 2024-09-17T00:49:50.686 DEBUG:teuthology.orchestra.run.smithi100:> /usr/sbin/getenforce 2024-09-17T00:49:50.713 INFO:teuthology.orchestra.run.smithi100.stdout:Permissive 2024-09-17T00:49:50.713 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi033.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi100.front.sepia.ceph.com': 'permissive'} 2024-09-17T00:49:50.714 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-17T00:49:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:49:50.757 DEBUG:teuthology.orchestra.run.smithi100:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-17T00:49:50.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:49:50.782 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-17T00:49:50.782 DEBUG:teuthology.orchestra.run.smithi033:> sudo /usr/sbin/setenforce permissive 2024-09-17T00:49:50.832 DEBUG:teuthology.orchestra.run.smithi100:> sudo /usr/sbin/setenforce permissive 2024-09-17T00:49:50.861 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-17T00:49:50.870 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-17T00:49:50.873 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-17T00:49:51.002 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-17T00:49:51.017 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-17T00:49:51.017 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi033.front.sepia.ceph.com,smithi100.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-17T00:53:40.117 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi033.front.sepia.ceph.com'), Remote(name='ubuntu@smithi100.front.sepia.ceph.com')] 2024-09-17T00:53:40.118 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-17T00:53:40.119 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:53:40.194 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-09-17T00:53:40.276 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-17T00:53:40.277 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-09-17T00:53:40.277 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi100.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-17T00:53:40.355 DEBUG:teuthology.orchestra.run.smithi100:> true 2024-09-17T00:53:40.436 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi100.front.sepia.ceph.com' 2024-09-17T00:53:40.436 INFO:teuthology.run_tasks:Running task clock... 2024-09-17T00:53:40.447 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-17T00:53:40.447 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T00:53:40.448 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T00:53:40.450 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-17T00:53:40.450 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T00:53:40.478 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T00:53:40.493 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T00:53:40.506 INFO:teuthology.orchestra.run.smithi100.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-17T00:53:40.518 INFO:teuthology.orchestra.run.smithi033.stderr:sudo: ntpd: command not found 2024-09-17T00:53:40.521 INFO:teuthology.orchestra.run.smithi100.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-17T00:53:40.529 INFO:teuthology.orchestra.run.smithi033.stdout:506 Cannot talk to daemon 2024-09-17T00:53:40.544 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T00:53:40.545 INFO:teuthology.orchestra.run.smithi100.stderr:sudo: ntpd: command not found 2024-09-17T00:53:40.556 INFO:teuthology.orchestra.run.smithi100.stdout:506 Cannot talk to daemon 2024-09-17T00:53:40.558 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T00:53:40.570 INFO:teuthology.orchestra.run.smithi100.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-17T00:53:40.584 INFO:teuthology.orchestra.run.smithi100.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-17T00:53:40.617 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-09-17T00:53:40.620 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T00:53:40.620 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-09-17T00:53:40.620 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.620 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.620 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.620 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.645 INFO:teuthology.orchestra.run.smithi100.stderr:bash: line 1: ntpq: command not found 2024-09-17T00:53:40.648 INFO:teuthology.orchestra.run.smithi100.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T00:53:40.649 INFO:teuthology.orchestra.run.smithi100.stdout:=============================================================================== 2024-09-17T00:53:40.649 INFO:teuthology.orchestra.run.smithi100.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.649 INFO:teuthology.orchestra.run.smithi100.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.649 INFO:teuthology.orchestra.run.smithi100.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.649 INFO:teuthology.orchestra.run.smithi100.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:53:40.650 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-17T00:53:40.747 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'} 2024-09-17T00:53:40.747 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-17T00:53:40.748 INFO:tasks.cephadm:Cluster fsid is 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:53:40.748 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-17T00:53:40.748 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-17T00:53:40.749 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi033': '172.21.15.33', 'mon.smithi100': '172.21.15.100'} 2024-09-17T00:53:40.749 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-17T00:53:40.749 DEBUG:teuthology.orchestra.run.smithi033:> sudo hostname $(hostname -s) 2024-09-17T00:53:40.776 DEBUG:teuthology.orchestra.run.smithi100:> sudo hostname $(hostname -s) 2024-09-17T00:53:40.802 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-17T00:53:40.803 DEBUG:teuthology.orchestra.run.smithi033:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T00:53:40.947 INFO:teuthology.orchestra.run.smithi033.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 00:53 /home/ubuntu/cephtest/cephadm 2024-09-17T00:53:40.948 DEBUG:teuthology.orchestra.run.smithi100:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-17T00:53:41.074 INFO:teuthology.orchestra.run.smithi100.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 17 00:53 /home/ubuntu/cephtest/cephadm 2024-09-17T00:53:41.074 DEBUG:teuthology.orchestra.run.smithi033:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T00:53:41.091 DEBUG:teuthology.orchestra.run.smithi100:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-17T00:53:41.115 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-17T00:53:41.115 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T00:53:41.134 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-17T00:53:42.721 INFO:teuthology.orchestra.run.smithi033.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T00:53:42.834 INFO:teuthology.orchestra.run.smithi100.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T00:53:54.368 INFO:teuthology.orchestra.run.smithi100.stdout:{ 2024-09-17T00:53:54.368 INFO:teuthology.orchestra.run.smithi100.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T00:53:54.368 INFO:teuthology.orchestra.run.smithi100.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T00:53:54.369 INFO:teuthology.orchestra.run.smithi100.stdout: "repo_digests": [ 2024-09-17T00:53:54.369 INFO:teuthology.orchestra.run.smithi100.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T00:53:54.369 INFO:teuthology.orchestra.run.smithi100.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T00:53:54.369 INFO:teuthology.orchestra.run.smithi100.stdout: ] 2024-09-17T00:53:54.369 INFO:teuthology.orchestra.run.smithi100.stdout:} 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout: "repo_digests": [ 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-17T00:53:55.531 INFO:teuthology.orchestra.run.smithi033.stdout: ] 2024-09-17T00:53:55.532 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T00:53:55.550 DEBUG:teuthology.orchestra.run.smithi033:> sudo mkdir -p /etc/ceph 2024-09-17T00:53:55.602 DEBUG:teuthology.orchestra.run.smithi100:> sudo mkdir -p /etc/ceph 2024-09-17T00:53:55.636 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 777 /etc/ceph 2024-09-17T00:53:55.660 DEBUG:teuthology.orchestra.run.smithi100:> sudo chmod 777 /etc/ceph 2024-09-17T00:53:55.704 INFO:tasks.cephadm:Writing seed config... 2024-09-17T00:53:55.705 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-17T00:53:55.705 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-17T00:53:55.705 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-17T00:53:55.706 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-17T00:53:55.706 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-17T00:53:55.706 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-17T00:53:55.706 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-17T00:53:55.706 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-17T00:53:55.706 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:53:55.706 DEBUG:teuthology.orchestra.run.smithi033:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-17T00:53:55.722 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 47eab686-748f-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-17T00:53:55.722 DEBUG:teuthology.orchestra.run.smithi033:mon.smithi033> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service 2024-09-17T00:53:55.766 INFO:tasks.cephadm:Bootstrapping... 2024-09-17T00:53:55.767 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 47eab686-748f-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.33 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:53:55.961 INFO:teuthology.orchestra.run.smithi033.stderr:-------------------------------------------------------------------------------- 2024-09-17T00:53:55.962 INFO:teuthology.orchestra.run.smithi033.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '47eab686-748f-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.33', '--skip-admin-label'] 2024-09-17T00:53:55.984 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: 5.2.2 2024-09-17T00:53:55.986 INFO:teuthology.orchestra.run.smithi033.stderr:Verifying podman|docker is present... 2024-09-17T00:53:56.007 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: 5.2.2 2024-09-17T00:53:56.009 INFO:teuthology.orchestra.run.smithi033.stderr:Verifying lvm2 is present... 2024-09-17T00:53:56.009 INFO:teuthology.orchestra.run.smithi033.stderr:Verifying time synchronization is in place... 2024-09-17T00:53:56.016 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T00:53:56.023 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: inactive 2024-09-17T00:53:56.030 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: enabled 2024-09-17T00:53:56.036 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: active 2024-09-17T00:53:56.037 INFO:teuthology.orchestra.run.smithi033.stderr:Unit chronyd.service is enabled and running 2024-09-17T00:53:56.037 INFO:teuthology.orchestra.run.smithi033.stderr:Repeating the final host check... 2024-09-17T00:53:56.057 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: 5.2.2 2024-09-17T00:53:56.059 INFO:teuthology.orchestra.run.smithi033.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-17T00:53:56.059 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl is present 2024-09-17T00:53:56.059 INFO:teuthology.orchestra.run.smithi033.stderr:lvcreate is present 2024-09-17T00:53:56.065 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-17T00:53:56.072 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: inactive 2024-09-17T00:53:56.079 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: enabled 2024-09-17T00:53:56.085 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: active 2024-09-17T00:53:56.086 INFO:teuthology.orchestra.run.smithi033.stderr:Unit chronyd.service is enabled and running 2024-09-17T00:53:56.086 INFO:teuthology.orchestra.run.smithi033.stderr:Host looks OK 2024-09-17T00:53:56.086 INFO:teuthology.orchestra.run.smithi033.stderr:Cluster fsid: 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:53:56.087 INFO:teuthology.orchestra.run.smithi033.stderr:Acquiring lock 140105824520080 on /run/cephadm/47eab686-748f-11ef-bceb-c7b262605968.lock 2024-09-17T00:53:56.087 INFO:teuthology.orchestra.run.smithi033.stderr:Lock 140105824520080 acquired on /run/cephadm/47eab686-748f-11ef-bceb-c7b262605968.lock 2024-09-17T00:53:56.087 INFO:teuthology.orchestra.run.smithi033.stderr:Verifying IP 172.21.15.33 port 3300 ... 2024-09-17T00:53:56.088 INFO:teuthology.orchestra.run.smithi033.stderr:Verifying IP 172.21.15.33 port 6789 ... 2024-09-17T00:53:56.088 INFO:teuthology.orchestra.run.smithi033.stderr:Base mon IP is 172.21.15.33, final addrv is [v2:172.21.15.33:3300,v1:172.21.15.33:6789] 2024-09-17T00:53:56.090 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.33 metric 100 2024-09-17T00:53:56.090 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.33 metric 100 2024-09-17T00:53:56.092 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-17T00:53:56.092 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-17T00:53:56.092 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1796sec hoplimit 64 pref medium 2024-09-17T00:53:56.095 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-17T00:53:56.095 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-17T00:53:56.095 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T00:53:56.095 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-17T00:53:56.096 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fea5:d294/64 scope link 2024-09-17T00:53:56.096 INFO:teuthology.orchestra.run.smithi033.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-17T00:53:56.096 INFO:teuthology.orchestra.run.smithi033.stderr:Mon IP `172.21.15.33` is in CIDR network `172.21.0.0/20` 2024-09-17T00:53:56.096 INFO:teuthology.orchestra.run.smithi033.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-17T00:53:56.097 INFO:teuthology.orchestra.run.smithi033.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-17T00:53:56.201 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-17T00:53:56.670 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Getting image source signatures 2024-09-17T00:53:56.670 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-17T00:53:56.670 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-17T00:53:56.671 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-17T00:53:56.671 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-17T00:53:56.671 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-17T00:53:56.671 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T00:53:56.843 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: Writing manifest to image destination 2024-09-17T00:53:56.996 INFO:teuthology.orchestra.run.smithi033.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-17T00:53:57.419 INFO:teuthology.orchestra.run.smithi033.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T00:53:58.228 INFO:teuthology.orchestra.run.smithi033.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-17T00:53:58.228 INFO:teuthology.orchestra.run.smithi033.stderr:Extracting ceph user uid/gid from container image... 2024-09-17T00:53:58.661 INFO:teuthology.orchestra.run.smithi033.stderr:stat: 167 167 2024-09-17T00:54:02.604 INFO:teuthology.orchestra.run.smithi033.stderr:Creating initial keys... 2024-09-17T00:54:06.333 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph-authtool: AQAt0+hmBeJKHxAAcGF8Lm8VM1FcFKE9yTGFUA== 2024-09-17T00:54:12.090 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph-authtool: AQA00+hmJt2EABAAzAEIoT4BPrxT6uXBXTEnqg== 2024-09-17T00:54:13.037 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph-authtool: AQA00+hmPQsEOBAAvpSBQQBgjusHE+3EDGwi9w== 2024-09-17T00:54:13.618 INFO:teuthology.orchestra.run.smithi033.stderr:Creating initial monmap... 2024-09-17T00:54:14.043 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T00:54:14.043 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-17T00:54:14.043 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:14.043 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T00:54:14.525 INFO:teuthology.orchestra.run.smithi033.stderr:monmaptool for smithi033 [v2:172.21.15.33:3300,v1:172.21.15.33:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-17T00:54:14.525 INFO:teuthology.orchestra.run.smithi033.stderr:setting min_mon_release = octopus 2024-09-17T00:54:14.525 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/monmaptool: set fsid to 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:14.525 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-17T00:54:14.525 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:54:14.525 INFO:teuthology.orchestra.run.smithi033.stderr:Creating mon... 2024-09-17T00:54:15.530 INFO:teuthology.orchestra.run.smithi033.stderr:create mon.smithi033 on 2024-09-17T00:54:15.757 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-17T00:54:15.908 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-47eab686-748f-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-47eab686-748f-11ef-bceb-c7b262605968.target. 2024-09-17T00:54:15.909 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-47eab686-748f-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-47eab686-748f-11ef-bceb-c7b262605968.target. 2024-09-17T00:54:16.235 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Failed to reset failed state of unit ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service: Unit ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service not loaded. 2024-09-17T00:54:16.244 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Created symlink /etc/systemd/system/ceph-47eab686-748f-11ef-bceb-c7b262605968.target.wants/ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service → /etc/systemd/system/ceph-47eab686-748f-11ef-bceb-c7b262605968@.service. 2024-09-17T00:54:16.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:16 smithi033 podman[23393]: 2024-09-17 00:54:16.674288447 +0000 UTC m=+0.119512070 container create 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, ceph=True, version=8, io.openshift.expose-services=) 2024-09-17T00:54:17.071 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: disabled 2024-09-17T00:54:17.078 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: inactive 2024-09-17T00:54:17.079 INFO:teuthology.orchestra.run.smithi033.stderr:firewalld.service is not enabled 2024-09-17T00:54:17.079 INFO:teuthology.orchestra.run.smithi033.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T00:54:17.080 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for mon to start... 2024-09-17T00:54:17.080 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for mon... 2024-09-17T00:54:17.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:16 smithi033 podman[23393]: 2024-09-17 00:54:16.976528882 +0000 UTC m=+0.421752517 container init 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.openshift.expose-services=, name=centos-stream, io.openshift.tags=base centos centos-stream, release=754, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T00:54:17.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:16 smithi033 podman[23393]: 2024-09-17 00:54:16.980150396 +0000 UTC m=+0.425374020 container start 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, name=centos-stream, version=8, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, distribution-scope=public, vendor=Red Hat, Inc.) 2024-09-17T00:54:17.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:17 smithi033 ceph-mon[23413]: mkfs 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:17.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:17 smithi033 ceph-mon[23413]: mon.smithi033 is new leader, mons smithi033 in quorum (ranks 0) 2024-09-17T00:54:17.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:17 smithi033 bash[23393]: 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa 2024-09-17T00:54:17.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:17 smithi033 systemd[1]: Started Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: cluster: 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: id: 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: services: 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi033 (age 0.628938s) 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: data: 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-17T00:54:17.650 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-17T00:54:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-17T00:54:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: pgs: 2024-09-17T00:54:17.651 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:18.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:18 smithi033 ceph-mon[23413]: mon.smithi033 is new leader, mons smithi033 in quorum (ranks 0) 2024-09-17T00:54:18.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:18 smithi033 ceph-mon[23413]: monmap e1: 1 mons at {smithi033=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0]} 2024-09-17T00:54:18.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:18 smithi033 ceph-mon[23413]: fsmap 2024-09-17T00:54:18.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:18 smithi033 ceph-mon[23413]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T00:54:18.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:18 smithi033 ceph-mon[23413]: mgrmap e1: no daemons active 2024-09-17T00:54:18.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:18 smithi033 ceph-mon[23413]: from='client.? 172.21.15.33:0/270295831' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-17T00:54:18.216 INFO:teuthology.orchestra.run.smithi033.stderr:mon is available 2024-09-17T00:54:18.216 INFO:teuthology.orchestra.run.smithi033.stderr:Assimilating anything we can from ceph.conf... 2024-09-17T00:54:18.756 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:18.756 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: [global] 2024-09-17T00:54:18.756 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: fsid = 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:18.756 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.33:3300,v1:172.21.15.33:6789] 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: [mgr] 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: [osd] 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T00:54:18.757 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T00:54:19.294 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:19 smithi033 ceph-mon[23413]: from='client.? 172.21.15.33:0/540435912' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T00:54:19.294 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:19 smithi033 ceph-mon[23413]: from='client.? 172.21.15.33:0/540435912' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-17T00:54:19.297 INFO:teuthology.orchestra.run.smithi033.stderr:Generating new minimal ceph.conf... 2024-09-17T00:54:20.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:20 smithi033 ceph-mon[23413]: from='client.? 172.21.15.33:0/704463179' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:54:20.412 INFO:teuthology.orchestra.run.smithi033.stderr:Restarting the monitor... 2024-09-17T00:54:20.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:20 smithi033 systemd[1]: Stopping Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T00:54:20.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:20 smithi033 bash[23629]: Error: no container with name or ID "ceph-47eab686-748f-11ef-bceb-c7b262605968-mon.smithi033" found: no such container 2024-09-17T00:54:20.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:20 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[23409]: 2024-09-17T00:54:20.606+0000 7fe133411700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi033 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:54:20.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:20 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[23409]: 2024-09-17T00:54:20.606+0000 7fe133411700 -1 mon.smithi033@0(leader) e1 *** Got Signal Terminated *** 2024-09-17T00:54:21.063 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:20 smithi033 podman[23641]: 2024-09-17 00:54:20.791494089 +0000 UTC m=+0.316509685 container died 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, distribution-scope=public, release=754, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, maintainer=Guillaume Abrioux , architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=) 2024-09-17T00:54:21.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 podman[23641]: 2024-09-17 00:54:21.148072343 +0000 UTC m=+0.673087928 container cleanup 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, ceph=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, name=centos-stream) 2024-09-17T00:54:21.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 bash[23641]: ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033 2024-09-17T00:54:21.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 bash[23669]: Error: no container with name or ID "ceph-47eab686-748f-11ef-bceb-c7b262605968-mon.smithi033" found: no such container 2024-09-17T00:54:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 podman[23653]: 2024-09-17 00:54:21.540845586 +0000 UTC m=+0.742374952 container remove 5588ed51c4fd23709cc52ca1cc7bd7a5b0978a62c0c9b3e7f48c2cf04b6d19fa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , distribution-scope=public, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-17T00:54:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service: Deactivated successfully. 2024-09-17T00:54:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 systemd[1]: Stopped Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T00:54:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 systemd[1]: Starting Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T00:54:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 podman[23730]: 2024-09-17 00:54:21.783124775 +0000 UTC m=+0.109945258 container create 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, GIT_BRANCH=HEAD, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, vcs-type=git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-17T00:54:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:21 smithi033 podman[23730]: 2024-09-17 00:54:21.690922096 +0000 UTC m=+0.017742592 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-17T00:54:22.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 podman[23730]: 2024-09-17 00:54:22.319065629 +0000 UTC m=+0.645886110 container init 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, maintainer=Guillaume Abrioux , release=754, distribution-scope=public, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, name=centos-stream, io.openshift.tags=base centos centos-stream, ceph=True, vendor=Red Hat, Inc., RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64) 2024-09-17T00:54:22.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 podman[23730]: 2024-09-17 00:54:22.32366172 +0000 UTC m=+0.650482201 container start 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, release=754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, distribution-scope=public, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True) 2024-09-17T00:54:22.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T00:54:22.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T00:54:22.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: pidfile_write: ignore empty --pid-file 2024-09-17T00:54:22.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: load: jerasure load: lrc 2024-09-17T00:54:22.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: RocksDB version: 6.15.5 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Compile date Apr 18 2022 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: DB SUMMARY 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: DB Session ID: EOZTXTINRD1HIN53OHE9 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: CURRENT file: CURRENT 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi033/store.db dir, Total Num: 1, files: 000008.sst 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi033/store.db: 000010.log size: 82699 ; 2024-09-17T00:54:22.415 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.error_if_exists: 0 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.create_if_missing: 0 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.paranoid_checks: 1 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.env: 0x5611464c6860 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.fs: Posix File System 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.info_log: 0x56114732dfa0 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.statistics: (nil) 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.use_fsync: 0 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_log_file_size: 0 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T00:54:22.416 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.allow_fallocate: 1 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.use_direct_reads: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.db_log_dir: 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi033/store.db 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T00:54:22.417 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.write_buffer_manager: 0x56114741e330 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T00:54:22.418 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.unordered_write: 0 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T00:54:22.419 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.row_cache: None 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.wal_filter: None 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.preserve_deletes: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.two_write_queues: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.atomic_flush: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.log_readahead_size: 0 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T00:54:22.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_background_jobs: 2 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_background_compactions: -1 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_subcompactions: 1 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T00:54:22.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_open_files: -1 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_background_flushes: -1 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Compression algorithms supported: 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kZSTD supported: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kXpressCompression supported: 0 2024-09-17T00:54:22.422 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kLZ4Compression supported: 1 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kBZip2Compression supported: 0 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kZlibCompression supported: 1 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: kSnappyCompression supported: 1 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi033/store.db/MANIFEST-000009 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.merge_operator: 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_filter: None 2024-09-17T00:54:22.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5611472f9d00) 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: index_type: 0 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: data_block_index_type: 0 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: index_shortening: 1 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: hash_index_allow_collision: 1 2024-09-17T00:54:22.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: checksum: 1 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: no_block_cache: 0 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache: 0x561147364170 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache_options: 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: capacity : 536870912 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: num_shard_bits : 4 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: strict_capacity_limit : 0 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache_compressed: (nil) 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: persistent_cache: (nil) 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_size: 4096 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_size_deviation: 10 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_restart_interval: 16 2024-09-17T00:54:22.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: index_block_restart_interval: 1 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: metadata_block_size: 4096 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: partition_filters: 0 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: use_delta_encoding: 1 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: whole_key_filtering: 1 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: verify_compression: 0 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: format_version: 4 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: enable_index_compression: 1 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_align: 0 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T00:54:22.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression: NoCompression 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.num_levels: 7 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T00:54:22.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T00:54:22.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T00:54:22.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T00:54:22.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T00:54:22.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:54:22.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T00:54:22.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T00:54:22.429 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T00:54:22.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.table_properties_collectors: 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.inplace_update_support: 0 2024-09-17T00:54:22.431 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.bloom_locality: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.max_successive_merges: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.ttl: 2592000 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.enable_blob_files: false 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.min_blob_size: 0 2024-09-17T00:54:22.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi033/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726534462353543, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726534462354378, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79381, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77645, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71954, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726534462, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "270a5bd5-83dc-4816-9ae1-f053e3ce5b35", "db_session_id": "EOZTXTINRD1HIN53OHE9"}} 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726534462354775, "job": 1, "event": "recovery_finished"} 2024-09-17T00:54:22.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi033/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x56114734aa80 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: DB pointer 0x56114735a000 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** DB Stats ** 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-17T00:54:22.434 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T00:54:22.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative compaction: 0.00 GB write, 21.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval compaction: 0.00 GB write, 21.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.436 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 138.4 0.00 0.00 1 0.001 0 0 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative compaction: 0.00 GB write, 21.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:54:22.437 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: starting mon.smithi033 rank 0 at public addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] at bind addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi033 fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???) e1 preinit fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).mds e1 new map 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).mds e1 print_map 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: e1 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: legacy client fscid: -1 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T00:54:22.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: No filesystems configured 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mon.smithi033 is new leader, mons smithi033 in quorum (ranks 0) 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: monmap e1: 1 mons at {smithi033=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0]} 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: fsmap 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: osdmap e1: 0 total, 0 up, 0 in 2024-09-17T00:54:22.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 ceph-mon[23750]: mgrmap e1: no daemons active 2024-09-17T00:54:22.445 INFO:teuthology.orchestra.run.smithi033.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-17T00:54:22.670 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 bash[23730]: 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce 2024-09-17T00:54:22.670 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:22 smithi033 systemd[1]: Started Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T00:54:23.517 INFO:teuthology.orchestra.run.smithi033.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-17T00:54:23.517 INFO:teuthology.orchestra.run.smithi033.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:54:23.517 INFO:teuthology.orchestra.run.smithi033.stderr:Creating mgr... 2024-09-17T00:54:23.517 INFO:teuthology.orchestra.run.smithi033.stderr:Verifying port 9283 ... 2024-09-17T00:54:23.710 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Failed to reset failed state of unit ceph-47eab686-748f-11ef-bceb-c7b262605968@mgr.smithi033.illmog.service: Unit ceph-47eab686-748f-11ef-bceb-c7b262605968@mgr.smithi033.illmog.service not loaded. 2024-09-17T00:54:23.718 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: Created symlink /etc/systemd/system/ceph-47eab686-748f-11ef-bceb-c7b262605968.target.wants/ceph-47eab686-748f-11ef-bceb-c7b262605968@mgr.smithi033.illmog.service → /etc/systemd/system/ceph-47eab686-748f-11ef-bceb-c7b262605968@.service. 2024-09-17T00:54:24.219 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:23 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1759535899' entity='client.admin' 2024-09-17T00:54:24.448 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: disabled 2024-09-17T00:54:24.455 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: inactive 2024-09-17T00:54:24.456 INFO:teuthology.orchestra.run.smithi033.stderr:firewalld.service is not enabled 2024-09-17T00:54:24.456 INFO:teuthology.orchestra.run.smithi033.stderr:Not possible to enable service . firewalld.service is not available 2024-09-17T00:54:24.465 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: disabled 2024-09-17T00:54:24.472 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: inactive 2024-09-17T00:54:24.473 INFO:teuthology.orchestra.run.smithi033.stderr:firewalld.service is not enabled 2024-09-17T00:54:24.473 INFO:teuthology.orchestra.run.smithi033.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-17T00:54:24.473 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for mgr to start... 2024-09-17T00:54:24.473 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for mgr... 2024-09-17T00:54:25.047 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsid": "47eab686-748f-11ef-bceb-c7b262605968", 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "health": { 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 0 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "smithi033" 2024-09-17T00:54:25.048 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T00:54:25.049 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T00:54:25.054 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "available": false, 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "iostat", 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "nfs", 2024-09-17T00:54:25.055 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "restful" 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modified": "2024-09-17T00:54:17.023576+0000", 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T00:54:25.056 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:54:25.339 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:25 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/172714813' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T00:54:25.635 INFO:teuthology.orchestra.run.smithi033.stderr:mgr not available, waiting (1/15)... 2024-09-17T00:54:28.152 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsid": "47eab686-748f-11ef-bceb-c7b262605968", 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "health": { 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 0 2024-09-17T00:54:28.153 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "smithi033" 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T00:54:28.154 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T00:54:28.155 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T00:54:28.160 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T00:54:28.160 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T00:54:28.160 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.160 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:28.161 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "available": false, 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "iostat", 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "nfs", 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "restful" 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T00:54:28.162 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:28.163 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modified": "2024-09-17T00:54:17.023576+0000", 2024-09-17T00:54:28.163 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:28.163 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:28.163 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T00:54:28.163 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:54:28.612 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:28 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3471137987' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T00:54:28.724 INFO:teuthology.orchestra.run.smithi033.stderr:mgr not available, waiting (2/15)... 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: Activating manager daemon smithi033.illmog 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: mgrmap e2: smithi033.illmog(active, starting, since 0.00266035s) 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T00:54:30.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: Manager daemon smithi033.illmog is now available 2024-09-17T00:54:30.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:54:30.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:54:30.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' 2024-09-17T00:54:30.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' 2024-09-17T00:54:30.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:30 smithi033 ceph-mon[23750]: from='mgr.14100 172.21.15.33:0/2042026765' entity='mgr.smithi033.illmog' 2024-09-17T00:54:31.244 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:31.244 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:54:31.244 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsid": "47eab686-748f-11ef-bceb-c7b262605968", 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "health": { 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 0 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "smithi033" 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:31.245 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T00:54:31.246 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T00:54:31.249 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T00:54:31.250 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "available": false, 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "iostat", 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "nfs", 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "restful" 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.251 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T00:54:31.252 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:31.252 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modified": "2024-09-17T00:54:17.023576+0000", 2024-09-17T00:54:31.252 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:31.252 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:31.252 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T00:54:31.252 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:54:31.797 INFO:teuthology.orchestra.run.smithi033.stderr:mgr not available, waiting (3/15)... 2024-09-17T00:54:32.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:31 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2501308724' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T00:54:32.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:31 smithi033 ceph-mon[23750]: mgrmap e3: smithi033.illmog(active, since 1.00602s) 2024-09-17T00:54:34.458 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:34 smithi033 ceph-mon[23750]: mgrmap e4: smithi033.illmog(active, since 2s) 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsid": "47eab686-748f-11ef-bceb-c7b262605968", 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "health": { 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "checks": {}, 2024-09-17T00:54:34.461 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mutes": [] 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum": [ 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 0 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "smithi033" 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "monmap": { 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-17T00:54:34.462 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.463 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osdmap": { 2024-09-17T00:54:34.463 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:34.463 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-17T00:54:34.463 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-17T00:54:34.463 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-17T00:54:34.463 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-17T00:54:34.466 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-17T00:54:34.466 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-17T00:54:34.466 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.466 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgmap": { 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "fsmap": { 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-17T00:54:34.467 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "available": true, 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modules": [ 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "iostat", 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "nfs", 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "restful" 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ], 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "servicemap": { 2024-09-17T00:54:34.468 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-17T00:54:34.469 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "modified": "2024-09-17T00:54:17.023576+0000", 2024-09-17T00:54:34.469 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "services": {} 2024-09-17T00:54:34.469 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: }, 2024-09-17T00:54:34.469 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-17T00:54:34.469 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:54:35.002 INFO:teuthology.orchestra.run.smithi033.stderr:mgr is available 2024-09-17T00:54:35.428 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:35 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1412170666' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-17T00:54:35.624 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: [global] 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: fsid = 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: [mgr] 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: [osd] 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-17T00:54:35.625 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-17T00:54:36.192 INFO:teuthology.orchestra.run.smithi033.stderr:Enabling cephadm module... 2024-09-17T00:54:36.609 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:36 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/4010912422' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-17T00:54:37.621 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:37 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2568570908' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-17T00:54:38.448 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:38 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2568570908' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-17T00:54:38.448 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:38 smithi033 ceph-mon[23750]: mgrmap e5: smithi033.illmog(active, since 6s) 2024-09-17T00:54:38.692 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:54:38.692 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-17T00:54:38.692 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "available": true, 2024-09-17T00:54:38.692 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "active_name": "smithi033.illmog", 2024-09-17T00:54:38.692 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T00:54:38.692 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:54:39.264 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for the mgr to restart... 2024-09-17T00:54:39.264 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for mgr epoch 5... 2024-09-17T00:54:39.679 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:39 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3687909752' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T00:54:44.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:43 smithi033 ceph-mon[23750]: Active manager daemon smithi033.illmog restarted 2024-09-17T00:54:44.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:43 smithi033 ceph-mon[23750]: Activating manager daemon smithi033.illmog 2024-09-17T00:54:44.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:43 smithi033 ceph-mon[23750]: osdmap e2: 0 total, 0 up, 0 in 2024-09-17T00:54:44.954 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:54:44.954 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-17T00:54:44.954 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T00:54:44.954 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: mgrmap e6: smithi033.illmog(active, starting, since 0.0525769s) 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: Manager daemon smithi033.illmog is now available 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:45.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:54:45.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:54:45.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:54:45.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:54:45.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:44 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:54:45.518 INFO:teuthology.orchestra.run.smithi033.stderr:mgr epoch 5 is available 2024-09-17T00:54:45.518 INFO:teuthology.orchestra.run.smithi033.stderr:Setting orchestrator backend to cephadm... 2024-09-17T00:54:46.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: [17/Sep/2024:00:54:44] ENGINE Bus STARTING 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: mgrmap e7: smithi033.illmog(active, since 1.06176s) 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: [17/Sep/2024:00:54:45] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: [17/Sep/2024:00:54:45] ENGINE Bus STARTED 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:45 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:54:47.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:47 smithi033 ceph-mon[23750]: mgrmap e8: smithi033.illmog(active, since 2s) 2024-09-17T00:54:47.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:47 smithi033 ceph-mon[23750]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:47.318 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:47 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:47.318 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:47 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:54:47.318 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: value unchanged 2024-09-17T00:54:47.871 INFO:teuthology.orchestra.run.smithi033.stderr:Generating ssh key... 2024-09-17T00:54:48.612 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:48 smithi033 ceph-mon[23750]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:50.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:49 smithi033 ceph-mon[23750]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:50.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:49 smithi033 ceph-mon[23750]: Generating ssh key... 2024-09-17T00:54:50.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:49 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:50.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:49 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:50.039 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvbqCq8nzpzeYqukj0eV8vcBKQSiCtWY8/j1MsmwZHzWjh21c7sgJLEit5xhkT1EWUFcN1nGAcvGJ109vuBt1l/H91ApiNgQMH/azPKeSN+99+A+qZJnw5iSKq3NMPuiKALqN+dYkdD3exb8dE/YYLclPDWGuRSaZgDscrxycMEgiINVufpeDqeQvGSZM8HuPUJn4vErSphjAu1PM4CodG/TXGmuuUNs36rR4/1vkVpJcubK0uP4Iavknx3WzCA1sOdA9epDmZkS+ujmGBgoHjw7/rCffUlBVCuVDHLy4B4xnXnVWBo8uL2f12kEv0r5afwNMqCZoI6Su/G7YByXlbm0JixaK07nJV9r1m4PCSM9EKCSs4RxjiBrN1KKODHG6eZcOKFed6+vKQwPbyWZ9WEAuC+tUuif+DwRQUI43Qb7r+Y0/9b+Jbq9HoiiKF05gkfF7ScCfYqWTiRXi03eq2m1KmRn2nE77wK0yt5hrFgfc3SiFMNfYvMpymhDS9rF0= ceph-47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:54:50.601 INFO:teuthology.orchestra.run.smithi033.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-17T00:54:50.601 INFO:teuthology.orchestra.run.smithi033.stderr:Adding key to root@localhost authorized_keys... 2024-09-17T00:54:50.605 INFO:teuthology.orchestra.run.smithi033.stderr:Adding host smithi033... 2024-09-17T00:54:51.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:50 smithi033 ceph-mon[23750]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:52.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:51 smithi033 ceph-mon[23750]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi033", "addr": "172.21.15.33", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:52.145 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Added host 'smithi033' with addr '172.21.15.33' 2024-09-17T00:54:52.725 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying mon service with default placement... 2024-09-17T00:54:53.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:53 smithi033 ceph-mon[23750]: Deploying cephadm binary to smithi033 2024-09-17T00:54:53.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:53 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:53.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:53 smithi033 ceph-mon[23750]: Added host smithi033 2024-09-17T00:54:53.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:53 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:54:53.350 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-17T00:54:53.983 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying mgr service with default placement... 2024-09-17T00:54:54.609 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:54 smithi033 ceph-mon[23750]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:54.609 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:54 smithi033 ceph-mon[23750]: Saving service mon spec with placement count:5 2024-09-17T00:54:54.609 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:54 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:54.646 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-17T00:54:55.241 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying crash service with default placement... 2024-09-17T00:54:55.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:55 smithi033 ceph-mon[23750]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:55.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:55 smithi033 ceph-mon[23750]: Saving service mgr spec with placement count:2 2024-09-17T00:54:55.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:55 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:55.906 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-17T00:54:56.365 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying prometheus service with default placement... 2024-09-17T00:54:57.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:56 smithi033 ceph-mon[23750]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:57.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:56 smithi033 ceph-mon[23750]: Saving service crash spec with placement * 2024-09-17T00:54:57.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:56 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:57.172 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-17T00:54:58.150 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying grafana service with default placement... 2024-09-17T00:54:58.478 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:58 smithi033 ceph-mon[23750]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:58.478 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:58 smithi033 ceph-mon[23750]: Saving service prometheus spec with placement count:1 2024-09-17T00:54:58.478 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:58 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:58.478 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:58 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:58.478 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:58 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:54:58.804 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-17T00:54:59.451 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying node-exporter service with default placement... 2024-09-17T00:54:59.910 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:59 smithi033 ceph-mon[23750]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:54:59.910 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:59 smithi033 ceph-mon[23750]: Saving service grafana spec with placement count:1 2024-09-17T00:54:59.910 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:54:59 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:55:00.041 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-17T00:55:00.869 INFO:teuthology.orchestra.run.smithi033.stderr:Deploying alertmanager service with default placement... 2024-09-17T00:55:01.131 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:01 smithi033 ceph-mon[23750]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:01.131 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:01 smithi033 ceph-mon[23750]: Saving service node-exporter spec with placement * 2024-09-17T00:55:01.132 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:01 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:55:01.132 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:01 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:55:01.576 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-17T00:55:02.508 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:02 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:55:02.508 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:02 smithi033 ceph-mon[23750]: from='mgr.14120 172.21.15.33:0/423790337' entity='mgr.smithi033.illmog' 2024-09-17T00:55:03.786 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:03 smithi033 ceph-mon[23750]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:03.787 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:03 smithi033 ceph-mon[23750]: Saving service alertmanager spec with placement count:1 2024-09-17T00:55:03.787 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:03 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/494476592' entity='client.admin' 2024-09-17T00:55:05.010 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:04 smithi033 ceph-mon[23750]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:05.010 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:04 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3322690114' entity='client.admin' 2024-09-17T00:55:05.013 INFO:teuthology.orchestra.run.smithi033.stderr:Enabling the dashboard module... 2024-09-17T00:55:06.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:05 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/609241308' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-17T00:55:07.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:06 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/609241308' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-17T00:55:07.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:06 smithi033 ceph-mon[23750]: mgrmap e9: smithi033.illmog(active, since 22s) 2024-09-17T00:55:07.320 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:55:07.320 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-17T00:55:07.320 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "available": true, 2024-09-17T00:55:07.320 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "active_name": "smithi033.illmog", 2024-09-17T00:55:07.320 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-17T00:55:07.320 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:55:07.869 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for the mgr to restart... 2024-09-17T00:55:07.870 INFO:teuthology.orchestra.run.smithi033.stderr:Waiting for mgr epoch 9... 2024-09-17T00:55:08.194 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:07 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2311527857' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-17T00:55:13.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: Active manager daemon smithi033.illmog restarted 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: Activating manager daemon smithi033.illmog 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: osdmap e3: 0 total, 0 up, 0 in 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: mgrmap e10: smithi033.illmog(active, starting, since 0.0524498s) 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: Manager daemon smithi033.illmog is now available 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:55:13.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:55:13.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:55:13.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:13 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:55:13.391 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: { 2024-09-17T00:55:13.391 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-17T00:55:13.392 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: "initialized": true 2024-09-17T00:55:13.392 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: } 2024-09-17T00:55:14.096 INFO:teuthology.orchestra.run.smithi033.stderr:mgr epoch 9 is available 2024-09-17T00:55:14.096 INFO:teuthology.orchestra.run.smithi033.stderr:Generating a dashboard self-signed certificate... 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: [17/Sep/2024:00:55:13] ENGINE Bus STARTING 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: mgrmap e11: smithi033.illmog(active, since 1.05557s) 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: [17/Sep/2024:00:55:13] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: [17/Sep/2024:00:55:13] ENGINE Bus STARTED 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:14.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:14 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:14.896 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-17T00:55:15.512 INFO:teuthology.orchestra.run.smithi033.stderr:Creating initial admin user... 2024-09-17T00:55:15.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:15 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:15.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:15 smithi033 ceph-mon[23750]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:15.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:15 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:15.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:15 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:15.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:15 smithi033 ceph-mon[23750]: mgrmap e12: smithi033.illmog(active, since 2s) 2024-09-17T00:55:16.566 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$4x6H2CSulgFlMKuptvlOW.MGDYYoGQB8yPlapJA.TrNGvPa8eyP9C", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726534516, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-17T00:55:17.257 INFO:teuthology.orchestra.run.smithi033.stderr:Fetching dashboard port number... 2024-09-17T00:55:17.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:17 smithi033 ceph-mon[23750]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:17.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:17 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:17.896 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: 8443 2024-09-17T00:55:18.610 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: disabled 2024-09-17T00:55:18.618 INFO:teuthology.orchestra.run.smithi033.stderr:systemctl: inactive 2024-09-17T00:55:18.619 INFO:teuthology.orchestra.run.smithi033.stderr:firewalld.service is not enabled 2024-09-17T00:55:18.619 INFO:teuthology.orchestra.run.smithi033.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-17T00:55:18.621 INFO:teuthology.orchestra.run.smithi033.stderr:Ceph Dashboard is now available at: 2024-09-17T00:55:18.621 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:18.621 INFO:teuthology.orchestra.run.smithi033.stderr: URL: https://smithi033.front.sepia.ceph.com:8443/ 2024-09-17T00:55:18.622 INFO:teuthology.orchestra.run.smithi033.stderr: User: admin 2024-09-17T00:55:18.622 INFO:teuthology.orchestra.run.smithi033.stderr: Password: 31j2m0hfw8 2024-09-17T00:55:18.622 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:18.622 INFO:teuthology.orchestra.run.smithi033.stderr:Enabling autotune for osd_memory_target 2024-09-17T00:55:18.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:18 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3647281479' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-17T00:55:19.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:19 smithi033 ceph-mon[23750]: mgrmap e13: smithi033.illmog(active, since 6s) 2024-09-17T00:55:20.731 INFO:teuthology.orchestra.run.smithi033.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-17T00:55:21.380 INFO:teuthology.orchestra.run.smithi033.stderr:You can access the Ceph CLI with: 2024-09-17T00:55:21.380 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:21.380 INFO:teuthology.orchestra.run.smithi033.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: ceph telemetry on 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr:For more information see: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T00:55:21.381 INFO:teuthology.orchestra.run.smithi033.stderr:Bootstrap complete. 2024-09-17T00:55:21.419 INFO:tasks.cephadm:Fetching config... 2024-09-17T00:55:21.420 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:55:21.420 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-17T00:55:21.438 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-17T00:55:21.438 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:55:21.438 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-17T00:55:21.496 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-17T00:55:21.496 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:55:21.496 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/keyring of=/dev/stdout 2024-09-17T00:55:21.564 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-17T00:55:21.564 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:55:21.564 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-17T00:55:21.622 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-17T00:55:21.623 DEBUG:teuthology.orchestra.run.smithi033:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvbqCq8nzpzeYqukj0eV8vcBKQSiCtWY8/j1MsmwZHzWjh21c7sgJLEit5xhkT1EWUFcN1nGAcvGJ109vuBt1l/H91ApiNgQMH/azPKeSN+99+A+qZJnw5iSKq3NMPuiKALqN+dYkdD3exb8dE/YYLclPDWGuRSaZgDscrxycMEgiINVufpeDqeQvGSZM8HuPUJn4vErSphjAu1PM4CodG/TXGmuuUNs36rR4/1vkVpJcubK0uP4Iavknx3WzCA1sOdA9epDmZkS+ujmGBgoHjw7/rCffUlBVCuVDHLy4B4xnXnVWBo8uL2f12kEv0r5afwNMqCZoI6Su/G7YByXlbm0JixaK07nJV9r1m4PCSM9EKCSs4RxjiBrN1KKODHG6eZcOKFed6+vKQwPbyWZ9WEAuC+tUuif+DwRQUI43Qb7r+Y0/9b+Jbq9HoiiKF05gkfF7ScCfYqWTiRXi03eq2m1KmRn2nE77wK0yt5hrFgfc3SiFMNfYvMpymhDS9rF0= ceph-47eab686-748f-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T00:55:21.711 INFO:teuthology.orchestra.run.smithi033.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvbqCq8nzpzeYqukj0eV8vcBKQSiCtWY8/j1MsmwZHzWjh21c7sgJLEit5xhkT1EWUFcN1nGAcvGJ109vuBt1l/H91ApiNgQMH/azPKeSN+99+A+qZJnw5iSKq3NMPuiKALqN+dYkdD3exb8dE/YYLclPDWGuRSaZgDscrxycMEgiINVufpeDqeQvGSZM8HuPUJn4vErSphjAu1PM4CodG/TXGmuuUNs36rR4/1vkVpJcubK0uP4Iavknx3WzCA1sOdA9epDmZkS+ujmGBgoHjw7/rCffUlBVCuVDHLy4B4xnXnVWBo8uL2f12kEv0r5afwNMqCZoI6Su/G7YByXlbm0JixaK07nJV9r1m4PCSM9EKCSs4RxjiBrN1KKODHG6eZcOKFed6+vKQwPbyWZ9WEAuC+tUuif+DwRQUI43Qb7r+Y0/9b+Jbq9HoiiKF05gkfF7ScCfYqWTiRXi03eq2m1KmRn2nE77wK0yt5hrFgfc3SiFMNfYvMpymhDS9rF0= ceph-47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:55:21.724 DEBUG:teuthology.orchestra.run.smithi100:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvbqCq8nzpzeYqukj0eV8vcBKQSiCtWY8/j1MsmwZHzWjh21c7sgJLEit5xhkT1EWUFcN1nGAcvGJ109vuBt1l/H91ApiNgQMH/azPKeSN+99+A+qZJnw5iSKq3NMPuiKALqN+dYkdD3exb8dE/YYLclPDWGuRSaZgDscrxycMEgiINVufpeDqeQvGSZM8HuPUJn4vErSphjAu1PM4CodG/TXGmuuUNs36rR4/1vkVpJcubK0uP4Iavknx3WzCA1sOdA9epDmZkS+ujmGBgoHjw7/rCffUlBVCuVDHLy4B4xnXnVWBo8uL2f12kEv0r5afwNMqCZoI6Su/G7YByXlbm0JixaK07nJV9r1m4PCSM9EKCSs4RxjiBrN1KKODHG6eZcOKFed6+vKQwPbyWZ9WEAuC+tUuif+DwRQUI43Qb7r+Y0/9b+Jbq9HoiiKF05gkfF7ScCfYqWTiRXi03eq2m1KmRn2nE77wK0yt5hrFgfc3SiFMNfYvMpymhDS9rF0= ceph-47eab686-748f-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-17T00:55:21.870 INFO:teuthology.orchestra.run.smithi100.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCvbqCq8nzpzeYqukj0eV8vcBKQSiCtWY8/j1MsmwZHzWjh21c7sgJLEit5xhkT1EWUFcN1nGAcvGJ109vuBt1l/H91ApiNgQMH/azPKeSN+99+A+qZJnw5iSKq3NMPuiKALqN+dYkdD3exb8dE/YYLclPDWGuRSaZgDscrxycMEgiINVufpeDqeQvGSZM8HuPUJn4vErSphjAu1PM4CodG/TXGmuuUNs36rR4/1vkVpJcubK0uP4Iavknx3WzCA1sOdA9epDmZkS+ujmGBgoHjw7/rCffUlBVCuVDHLy4B4xnXnVWBo8uL2f12kEv0r5afwNMqCZoI6Su/G7YByXlbm0JixaK07nJV9r1m4PCSM9EKCSs4RxjiBrN1KKODHG6eZcOKFed6+vKQwPbyWZ9WEAuC+tUuif+DwRQUI43Qb7r+Y0/9b+Jbq9HoiiKF05gkfF7ScCfYqWTiRXi03eq2m1KmRn2nE77wK0yt5hrFgfc3SiFMNfYvMpymhDS9rF0= ceph-47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:55:21.885 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-17T00:55:22.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:21 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3424261584' entity='client.admin' 2024-09-17T00:55:24.905 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-17T00:55:24.905 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-17T00:55:25.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:24 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3831496570' entity='client.admin' 2024-09-17T00:55:25.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:24 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:25.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:24 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:55:25.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:24 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:26.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:25 smithi033 ceph-mon[23750]: Deploying daemon alertmanager.smithi033 on smithi033 2024-09-17T00:55:28.674 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi100 2024-09-17T00:55:28.674 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-09-17T00:55:28.674 DEBUG:teuthology.orchestra.run.smithi100:> dd of=/etc/ceph/ceph.conf 2024-09-17T00:55:28.693 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-09-17T00:55:28.693 DEBUG:teuthology.orchestra.run.smithi100:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:55:28.750 INFO:tasks.cephadm:Adding host smithi100 to orchestrator... 2024-09-17T00:55:28.751 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch host add smithi100 2024-09-17T00:55:29.063 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:28 smithi033 ceph-mon[23750]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:29.064 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:28 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:32.312 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:32 smithi033 ceph-mon[23750]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi100", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:33.130 INFO:teuthology.orchestra.run.smithi033.stdout:Added host 'smithi100' with addr '172.21.15.100' 2024-09-17T00:55:33.486 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:33 smithi033 ceph-mon[23750]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:33.487 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:33 smithi033 ceph-mon[23750]: Deploying cephadm binary to smithi100 2024-09-17T00:55:33.487 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:33 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:34.494 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:34 smithi033 ceph-mon[23750]: Added host smithi100 2024-09-17T00:55:34.744 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-17T00:55:35.633 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:35 smithi033 ceph-mon[23750]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:36.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:36 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:36.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:36 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:55:36.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:36 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T00:55:36.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:36 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:55:36.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:36 smithi033 ceph-mon[23750]: Deploying daemon crash.smithi033 on smithi033 2024-09-17T00:55:37.852 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:37 smithi033 ceph-mon[23750]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:37.852 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:37 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:37.852 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:55:37.852 INFO:teuthology.orchestra.run.smithi033.stdout:[{"addr": "172.21.15.33", "hostname": "smithi033", "labels": [], "status": ""}, {"addr": "172.21.15.100", "hostname": "smithi100", "labels": [], "status": ""}] 2024-09-17T00:55:38.494 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-17T00:55:38.494 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-17T00:55:38.815 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:38 smithi033 ceph-mon[23750]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T00:55:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:39 smithi033 ceph-mon[23750]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:39 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:39 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:39 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:39 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:55:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:39 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:40.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:40 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:55:40.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:40 smithi033 ceph-mon[23750]: Deploying daemon grafana.smithi033 on smithi033 2024-09-17T00:55:41.526 INFO:teuthology.orchestra.run.smithi033.stderr:adjusted tunables profile to default 2024-09-17T00:55:41.846 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:41 smithi033 ceph-mon[23750]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:41.846 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:41 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3075887025' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T00:55:42.182 INFO:tasks.cephadm:Adding mon.smithi033 on smithi033 2024-09-17T00:55:42.182 INFO:tasks.cephadm:Adding mon.smithi100 on smithi100 2024-09-17T00:55:42.182 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi033:172.21.15.33=smithi033;smithi100:172.21.15.100=smithi100' 2024-09-17T00:55:42.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:42 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3075887025' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T00:55:42.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:42 smithi033 ceph-mon[23750]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T00:55:42.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:42 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:43.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:43 smithi033 ceph-mon[23750]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:43.949 INFO:teuthology.orchestra.run.smithi100.stdout:Scheduled mon update... 2024-09-17T00:55:44.535 DEBUG:teuthology.orchestra.run.smithi100:mon.smithi100> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi100.service 2024-09-17T00:55:44.537 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:55:44.537 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:55:45.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:44 smithi033 ceph-mon[23750]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi033:172.21.15.33=smithi033;smithi100:172.21.15.100=smithi100", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:55:45.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:44 smithi033 ceph-mon[23750]: Saving service mon spec with placement smithi033:172.21.15.33=smithi033;smithi100:172.21.15.100=smithi100;count:2 2024-09-17T00:55:45.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:44 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:46.361 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:55:46.361 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:55:46.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:45 smithi033 ceph-mon[23750]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:46.363 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:55:47.344 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:46 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/1959210131' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:55:47.964 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:55:47.964 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:55:48.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:47 smithi033 ceph-mon[23750]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:49.728 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:55:49.728 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:55:49.730 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:55:50.205 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:49 smithi033 ceph-mon[23750]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:50.205 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:49 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/2040450089' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:55:51.291 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:55:51.291 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:55:51.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:51 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:51.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:51 smithi033 ceph-mon[23750]: Deploying daemon node-exporter.smithi033 on smithi033 2024-09-17T00:55:51.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:51 smithi033 ceph-mon[23750]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:53.149 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:55:53.149 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:55:53.151 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:55:53.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:53 smithi033 ceph-mon[23750]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:55:53.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:53 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:53.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:53 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/2448526255' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:55:53.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:53 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:55:53.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:53 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T00:55:54.656 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:55:54.656 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:55:55.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:55 smithi033 ceph-mon[23750]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T00:55:55.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:55 smithi033 ceph-mon[23750]: mgrmap e14: smithi033.illmog(active, since 41s) 2024-09-17T00:55:56.534 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:55:56.534 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:55:56.536 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:55:56.856 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:56 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/1836406019' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:55:58.031 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:55:58.031 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:55:59.748 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:55:59.748 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:55:59.750 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:56:00.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:55:59 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/2708549626' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:00.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:00 smithi033 ceph-mon[23750]: Active manager daemon smithi033.illmog restarted 2024-09-17T00:56:00.858 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:00 smithi033 ceph-mon[23750]: Activating manager daemon smithi033.illmog 2024-09-17T00:56:00.858 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:00 smithi033 ceph-mon[23750]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T00:56:01.337 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:56:01.337 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:56:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: mgrmap e15: smithi033.illmog(active, starting, since 0.222946s) 2024-09-17T00:56:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:56:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T00:56:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:56:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:56:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: Manager daemon smithi033.illmog is now available 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:02.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:03.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:02 smithi033 ceph-mon[23750]: [17/Sep/2024:00:56:01] ENGINE Bus STARTING 2024-09-17T00:56:03.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:02 smithi033 ceph-mon[23750]: [17/Sep/2024:00:56:01] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T00:56:03.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:02 smithi033 ceph-mon[23750]: [17/Sep/2024:00:56:01] ENGINE Bus STARTED 2024-09-17T00:56:03.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:02 smithi033 ceph-mon[23750]: mgrmap e16: smithi033.illmog(active, since 1.2256s) 2024-09-17T00:56:03.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:03.186 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:56:03.186 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:56:03.189 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:56:04.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:03 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:04.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:03 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/4140647729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:04.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:03 smithi033 ceph-mon[23750]: mgrmap e17: smithi033.illmog(active, since 2s) 2024-09-17T00:56:04.809 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:56:04.810 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:56:05.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:05 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:05.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:05 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:05.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:05 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:56:05.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:05 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T00:56:05.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:05 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:56:05.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:05 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:07.266 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:56:07.266 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:56:07.268 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd/host:smithi100", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T00:56:07.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:07.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:07 smithi033 ceph-mon[23750]: Deploying daemon crash.smithi100 on smithi100 2024-09-17T00:56:08.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:08 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/3646805031' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:08.828 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:56:08.828 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:56:11.105 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:56:11.106 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:56:11.108 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: Deploying daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T00:56:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:12 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/3992392268' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:12.762 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:56:12.763 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:56:14.585 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:56:14.586 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:56:14.586 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:56:15.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:14 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/2196222971' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:16.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:56:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:15 smithi033 ceph-mon[23750]: Deploying daemon mon.smithi100 on smithi100 2024-09-17T00:56:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:16.478 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:56:16.478 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:56:19.511 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:56:19.511 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":1,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:54:13.960446Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-17T00:56:19.511 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:19 smithi100 podman[25796]: 2024-09-17 00:56:19.411499293 +0000 UTC m=+0.017153202 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-09-17T00:56:19.512 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 1 2024-09-17T00:56:19.832 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:19 smithi100 podman[25796]: 2024-09-17 00:56:19.538440692 +0000 UTC m=+0.144094592 container create f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, architecture=x86_64, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, RELEASE=HEAD, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, version=8, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T00:56:19.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:19 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/1067105974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.524 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 podman[25796]: 2024-09-17 00:56:20.427907373 +0000 UTC m=+1.033561284 container init f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, ceph=True, release=754, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, name=centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux ) 2024-09-17T00:56:20.525 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 podman[25796]: 2024-09-17 00:56:20.432008834 +0000 UTC m=+1.037662734 container start f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, release=754, ceph=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, version=8) 2024-09-17T00:56:20.525 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T00:56:20.525 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-17T00:56:20.525 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pidfile_write: ignore empty --pid-file 2024-09-17T00:56:20.525 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: load: jerasure load: lrc 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: RocksDB version: 6.15.5 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Compile date Apr 18 2022 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: DB SUMMARY 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: DB Session ID: HY0X5KIGPBET9SQVHICD 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: CURRENT file: CURRENT 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi100/store.db dir, Total Num: 0, files: 2024-09-17T00:56:20.526 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi100/store.db: 000004.log size: 511 ; 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.error_if_exists: 0 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.create_if_missing: 0 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.paranoid_checks: 1 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.env: 0x560acd6c6860 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.fs: Posix File System 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.info_log: 0x560acf5e9fa0 2024-09-17T00:56:20.527 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.statistics: (nil) 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.use_fsync: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_log_file_size: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.allow_fallocate: 1 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.use_direct_reads: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T00:56:20.528 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.db_log_dir: 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi100/store.db 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.write_buffer_manager: 0x560acf6da330 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-17T00:56:20.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.unordered_write: 0 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.row_cache: None 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.wal_filter: None 2024-09-17T00:56:20.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.preserve_deletes: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.two_write_queues: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.atomic_flush: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T00:56:20.531 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T00:56:20.532 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.log_readahead_size: 0 2024-09-17T00:56:20.532 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T00:56:20.532 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T00:56:20.532 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T00:56:20.532 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T00:56:20.532 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T00:56:20.533 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T00:56:20.533 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_background_jobs: 2 2024-09-17T00:56:20.533 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_background_compactions: -1 2024-09-17T00:56:20.533 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_subcompactions: 1 2024-09-17T00:56:20.533 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T00:56:20.533 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T00:56:20.534 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T00:56:20.534 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T00:56:20.534 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T00:56:20.534 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T00:56:20.534 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T00:56:20.534 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_open_files: -1 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_background_flushes: -1 2024-09-17T00:56:20.535 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Compression algorithms supported: 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kZSTD supported: 0 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kXpressCompression supported: 0 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kLZ4Compression supported: 1 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kBZip2Compression supported: 0 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kZlibCompression supported: 1 2024-09-17T00:56:20.536 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: kSnappyCompression supported: 1 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi100/store.db/MANIFEST-000003 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.merge_operator: 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_filter: None 2024-09-17T00:56:20.537 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T00:56:20.538 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T00:56:20.538 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T00:56:20.538 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T00:56:20.538 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560acf5b5d00) 2024-09-17T00:56:20.538 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: cache_index_and_filter_blocks: 1 2024-09-17T00:56:20.539 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T00:56:20.539 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T00:56:20.539 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: pin_top_level_index_and_filter: 1 2024-09-17T00:56:20.539 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: index_type: 0 2024-09-17T00:56:20.539 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: data_block_index_type: 0 2024-09-17T00:56:20.539 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: index_shortening: 1 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: hash_index_allow_collision: 1 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: checksum: 1 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: no_block_cache: 0 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache: 0x560acf620170 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache_name: BinnedLRUCache 2024-09-17T00:56:20.540 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache_options: 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: capacity : 536870912 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: num_shard_bits : 4 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: strict_capacity_limit : 0 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: high_pri_pool_ratio: 0.000 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache_compressed: (nil) 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: persistent_cache: (nil) 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_size: 4096 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_size_deviation: 10 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_restart_interval: 16 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: index_block_restart_interval: 1 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: metadata_block_size: 4096 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: partition_filters: 0 2024-09-17T00:56:20.541 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: use_delta_encoding: 1 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: whole_key_filtering: 1 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: verify_compression: 0 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: read_amp_bytes_per_bit: 0 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: format_version: 4 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: enable_index_compression: 1 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_align: 0 2024-09-17T00:56:20.542 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T00:56:20.543 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T00:56:20.543 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression: NoCompression 2024-09-17T00:56:20.543 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T00:56:20.543 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T00:56:20.543 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T00:56:20.543 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.num_levels: 7 2024-09-17T00:56:20.544 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T00:56:20.544 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T00:56:20.544 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T00:56:20.544 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T00:56:20.544 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T00:56:20.544 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T00:56:20.545 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T00:56:20.545 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:56:20.545 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T00:56:20.545 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T00:56:20.545 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T00:56:20.545 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T00:56:20.546 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T00:56:20.546 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T00:56:20.546 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:56:20.546 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T00:56:20.548 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T00:56:20.549 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T00:56:20.549 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T00:56:20.549 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T00:56:20.549 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T00:56:20.549 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T00:56:20.549 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T00:56:20.550 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.arena_block_size: 4194304 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-17T00:56:20.551 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T00:56:20.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T00:56:20.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T00:56:20.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T00:56:20.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T00:56:20.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.table_properties_collectors: 2024-09-17T00:56:20.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.inplace_update_support: 0 2024-09-17T00:56:20.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T00:56:20.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T00:56:20.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T00:56:20.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T00:56:20.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.bloom_locality: 0 2024-09-17T00:56:20.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.max_successive_merges: 0 2024-09-17T00:56:20.555 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T00:56:20.555 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T00:56:20.555 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T00:56:20.555 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T00:56:20.555 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.ttl: 2592000 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.enable_blob_files: false 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.min_blob_size: 0 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T00:56:20.556 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi100/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726534580457853, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726534580458145, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726534580, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "6a270a39-4d88-40c4-bc0f-cd3b351d8e2f", "db_session_id": "HY0X5KIGPBET9SQVHICD"}} 2024-09-17T00:56:20.557 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726534580458443, "job": 1, "event": "recovery_finished"} 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi100/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x560acf606a80 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: DB pointer 0x560acf616000 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** DB Stats ** 2024-09-17T00:56:20.558 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.559 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** Compaction Stats [default] ** 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** Compaction Stats [default] ** 2024-09-17T00:56:20.561 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:56:20.561 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:56:20.561 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.561 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:56:20.561 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T00:56:20.561 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T00:56:20.562 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T00:56:20.562 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T00:56:20.562 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T00:56:20.562 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:56:20.562 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:56:20.562 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T00:56:20.563 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.563 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T00:56:20.563 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.563 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** Compaction Stats [default] ** 2024-09-17T00:56:20.563 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:56:20.563 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** Compaction Stats [default] ** 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:56:20.564 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.3 0.00 0.00 1 0.000 0 0 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100 does not exist in monmap, will attempt to join an existing cluster 2024-09-17T00:56:20.565 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: using public_addr v2:172.21.15.100:0/0 -> [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: starting mon.smithi100 rank -1 at public addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] at bind addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi100 fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(???) e0 preinit fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).mds e1 new map 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).mds e1 print_map 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: e1 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: legacy client fscid: -1 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: No filesystems configured 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-17T00:56:20.566 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:56:20.567 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Deploying daemon crash.smithi033 on smithi033 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Deploying daemon grafana.smithi033 on smithi033 2024-09-17T00:56:20.568 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3075887025' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3075887025' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: osdmap e4: 0 total, 0 up, 0 in 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi033:172.21.15.33=smithi033;smithi100:172.21.15.100=smithi100", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Saving service mon spec with placement smithi033:172.21.15.33=smithi033;smithi100:172.21.15.100=smithi100;count:2 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/1959210131' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/2040450089' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Deploying daemon node-exporter.smithi033 on smithi033 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/2448526255' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14162 172.21.15.33:0/2503381402' entity='mgr.smithi033.illmog' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mgrmap e14: smithi033.illmog(active, since 41s) 2024-09-17T00:56:20.570 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/1836406019' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/2708549626' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Active manager daemon smithi033.illmog restarted 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Activating manager daemon smithi033.illmog 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mgrmap e15: smithi033.illmog(active, starting, since 0.222946s) 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:56:20.571 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Manager daemon smithi033.illmog is now available 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.572 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: [17/Sep/2024:00:56:01] ENGINE Bus STARTING 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: [17/Sep/2024:00:56:01] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: [17/Sep/2024:00:56:01] ENGINE Bus STARTED 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mgrmap e16: smithi033.illmog(active, since 1.2256s) 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/4140647729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mgrmap e17: smithi033.illmog(active, since 2s) 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T00:56:20.573 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd/host:smithi100", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Deploying daemon crash.smithi100 on smithi100 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/3646805031' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.574 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Deploying daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/3992392268' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/2196222971' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.575 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: Deploying daemon mon.smithi100 on smithi100 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/1067105974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 ceph-mon[25838]: mon.smithi100@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 bash[25796]: f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 2024-09-17T00:56:20.576 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:20 smithi100 systemd[1]: Started Ceph mon.smithi100 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T00:56:21.255 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-17T00:56:21.256 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: mon.smithi033 calling monitor election 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: mon.smithi100 calling monitor election 2024-09-17T00:56:25.778 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: mon.smithi033 is new leader, mons smithi033,smithi100 in quorum (ranks 0,1) 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: monmap e2: 2 mons at {smithi033=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],smithi100=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0]} 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: fsmap 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: mgrmap e17: smithi033.illmog(active, since 24s) 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie started 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: overall HEALTH_OK 2024-09-17T00:56:25.779 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T00:56:25.780 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:25.780 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:56:25.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: mon.smithi033 calling monitor election 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: mon.smithi100 calling monitor election 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: mon.smithi033 is new leader, mons smithi033,smithi100 in quorum (ranks 0,1) 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: monmap e2: 2 mons at {smithi033=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],smithi100=[v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0]} 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: fsmap 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: osdmap e5: 0 total, 0 up, 0 in 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: mgrmap e17: smithi033.illmog(active, since 24s) 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: Standby manager daemon smithi100.qrwkie started 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: overall HEALTH_OK 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:25.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:25 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1950138080' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:56:26.268 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:56:26.268 INFO:teuthology.orchestra.run.smithi100.stdout:{"epoch":2,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","modified":"2024-09-17T00:56:20.484217Z","created":"2024-09-17T00:54:13.960446Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi033","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi100","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:3300","nonce":0},{"type":"v1","addr":"172.21.15.100:6789","nonce":0}]},"addr":"172.21.15.100:6789/0","public_addr":"172.21.15.100:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-17T00:56:26.269 INFO:teuthology.orchestra.run.smithi100.stderr:dumped monmap epoch 2 2024-09-17T00:56:26.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:26 smithi033 ceph-mon[23750]: Deploying daemon node-exporter.smithi100 on smithi100 2024-09-17T00:56:26.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:26 smithi033 ceph-mon[23750]: mgrmap e18: smithi033.illmog(active, since 24s), standbys: smithi100.qrwkie 2024-09-17T00:56:26.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:26 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T00:56:26.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:26 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:26.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:26 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/3788381394' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:26.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:26 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:26.939 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:26 smithi100 ceph-mon[25838]: Deploying daemon node-exporter.smithi100 on smithi100 2024-09-17T00:56:26.939 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:26 smithi100 ceph-mon[25838]: mgrmap e18: smithi033.illmog(active, since 24s), standbys: smithi100.qrwkie 2024-09-17T00:56:26.939 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:26 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T00:56:26.939 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:26 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:26.939 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:26 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/3788381394' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-17T00:56:26.939 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:26 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:56:26.961 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-17T00:56:26.961 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-17T00:56:27.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:27 smithi033 ceph-mon[23750]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:27.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:27 smithi100 ceph-mon[25838]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:28.772 INFO:teuthology.orchestra.run.smithi033.stdout:# minimal ceph.conf for 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:56:28.772 INFO:teuthology.orchestra.run.smithi033.stdout:[global] 2024-09-17T00:56:28.772 INFO:teuthology.orchestra.run.smithi033.stdout: fsid = 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T00:56:28.772 INFO:teuthology.orchestra.run.smithi033.stdout: mon_host = [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] 2024-09-17T00:56:29.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:28 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3107621564' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:29.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:28 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3107621564' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:29.325 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-17T00:56:29.325 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:56:29.325 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T00:56:29.352 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:56:29.352 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:56:29.419 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-09-17T00:56:29.420 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/ceph/ceph.conf 2024-09-17T00:56:29.450 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-09-17T00:56:29.450 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:56:29.517 INFO:tasks.cephadm:Deploying OSDs... 2024-09-17T00:56:29.517 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T00:56:29.517 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T00:56:29.533 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T00:56:29.534 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_1 2024-09-17T00:56:29.589 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T00:56:29.589 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:29.589 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-17T00:56:29.590 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:29.590 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:29.590 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-17 00:55:50.948496076 +0000 2024-09-17T00:56:29.590 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-17 00:53:20.281336559 +0000 2024-09-17T00:56:29.590 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-17 00:53:20.281336559 +0000 2024-09-17T00:56:29.590 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-17 00:53:20.281336559 +0000 2024-09-17T00:56:29.590 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T00:56:29.653 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-17T00:56:29.654 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-17T00:56:29.654 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000216601 s, 2.4 MB/s 2024-09-17T00:56:29.655 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T00:56:29.712 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_2 2024-09-17T00:56:29.768 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T00:56:29.768 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:29.768 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 685 Links: 1 2024-09-17T00:56:29.768 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:29.769 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:29.769 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-17 00:55:50.948496076 +0000 2024-09-17T00:56:29.769 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-17 00:53:20.612331211 +0000 2024-09-17T00:56:29.769 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-17 00:53:20.612331211 +0000 2024-09-17T00:56:29.769 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-17 00:53:20.612331211 +0000 2024-09-17T00:56:29.769 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T00:56:29.832 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-17T00:56:29.833 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-17T00:56:29.833 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.00019315 s, 2.7 MB/s 2024-09-17T00:56:29.834 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T00:56:29.891 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_3 2024-09-17T00:56:29.947 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T00:56:29.947 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-17 00:55:50.948496076 +0000 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-17 00:53:20.977325313 +0000 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-17 00:53:20.977325313 +0000 2024-09-17T00:56:29.948 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-17 00:53:20.977325313 +0000 2024-09-17T00:56:29.948 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T00:56:30.009 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:29 smithi033 ceph-mon[23750]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:30.009 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:29 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:30.009 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:29 smithi033 ceph-mon[23750]: Deploying daemon prometheus.smithi033 on smithi033 2024-09-17T00:56:30.011 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-17T00:56:30.011 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-17T00:56:30.011 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000179747 s, 2.8 MB/s 2024-09-17T00:56:30.012 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T00:56:30.070 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_4 2024-09-17T00:56:30.125 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T00:56:30.125 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:30.125 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 725 Links: 1 2024-09-17T00:56:30.125 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:30.126 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:30.126 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-17 00:55:50.949496058 +0000 2024-09-17T00:56:30.126 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-17 00:53:21.302320062 +0000 2024-09-17T00:56:30.126 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-17 00:53:21.302320062 +0000 2024-09-17T00:56:30.126 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-17 00:53:21.302320062 +0000 2024-09-17T00:56:30.126 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T00:56:30.189 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-17T00:56:30.190 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-17T00:56:30.190 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000197062 s, 2.6 MB/s 2024-09-17T00:56:30.191 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T00:56:30.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:29 smithi100 ceph-mon[25838]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:30.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:29 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:30.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:29 smithi100 ceph-mon[25838]: Deploying daemon prometheus.smithi033 on smithi033 2024-09-17T00:56:30.248 DEBUG:teuthology.orchestra.run.smithi100:> set -ex 2024-09-17T00:56:30.248 DEBUG:teuthology.orchestra.run.smithi100:> dd if=/scratch_devs of=/dev/stdout 2024-09-17T00:56:30.267 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-17T00:56:30.267 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_1 2024-09-17T00:56:30.325 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-17T00:56:30.325 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:30.325 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-17T00:56:30.325 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:30.326 INFO:teuthology.orchestra.run.smithi100.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:30.326 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-09-17 00:56:26.056661986 +0000 2024-09-17T00:56:30.326 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-09-17 00:53:16.856124979 +0000 2024-09-17T00:56:30.326 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-09-17 00:53:16.856124979 +0000 2024-09-17T00:56:30.326 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: 2024-09-17 00:53:16.856124979 +0000 2024-09-17T00:56:30.326 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-17T00:56:30.392 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-09-17T00:56:30.393 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-09-17T00:56:30.393 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000234544 s, 2.2 MB/s 2024-09-17T00:56:30.394 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-17T00:56:30.454 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_2 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-09-17 00:56:26.056661986 +0000 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-09-17 00:53:17.165116653 +0000 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-09-17 00:53:17.165116653 +0000 2024-09-17T00:56:30.512 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: 2024-09-17 00:53:17.165116653 +0000 2024-09-17T00:56:30.512 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-17T00:56:30.577 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-09-17T00:56:30.578 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-09-17T00:56:30.578 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000205507 s, 2.5 MB/s 2024-09-17T00:56:30.579 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-17T00:56:30.637 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_3 2024-09-17T00:56:30.693 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-09-17 00:56:26.057661959 +0000 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-09-17 00:53:17.484108056 +0000 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-09-17 00:53:17.484108056 +0000 2024-09-17T00:56:30.694 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: 2024-09-17 00:53:17.484108056 +0000 2024-09-17T00:56:30.695 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-17T00:56:30.758 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-09-17T00:56:30.758 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-09-17T00:56:30.758 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000180923 s, 2.8 MB/s 2024-09-17T00:56:30.759 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-17T00:56:30.816 DEBUG:teuthology.orchestra.run.smithi100:> stat /dev/vg_nvme/lv_4 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout:Context: system_u:object_r:device_t:s0 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout:Access: 2024-09-17 00:56:26.057661959 +0000 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout:Modify: 2024-09-17 00:53:17.824098894 +0000 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout:Change: 2024-09-17 00:53:17.824098894 +0000 2024-09-17T00:56:30.872 INFO:teuthology.orchestra.run.smithi100.stdout: Birth: 2024-09-17 00:53:17.824098894 +0000 2024-09-17T00:56:30.873 DEBUG:teuthology.orchestra.run.smithi100:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-17T00:56:30.936 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records in 2024-09-17T00:56:30.936 INFO:teuthology.orchestra.run.smithi100.stderr:1+0 records out 2024-09-17T00:56:30.936 INFO:teuthology.orchestra.run.smithi100.stderr:512 bytes copied, 0.000190065 s, 2.7 MB/s 2024-09-17T00:56:30.937 DEBUG:teuthology.orchestra.run.smithi100:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-17T00:56:30.993 INFO:tasks.cephadm:Deploying osd.0 on smithi033 with /dev/vg_nvme/lv_4... 2024-09-17T00:56:30.993 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T00:56:32.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:31 smithi033 ceph-mon[23750]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:32.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:31 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:31 smithi100 ceph-mon[25838]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:31 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:33 smithi100 ceph-mon[25838]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:33.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:33 smithi033 ceph-mon[23750]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:36.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:35 smithi033 ceph-mon[23750]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:35 smithi100 ceph-mon[25838]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:37 smithi100 ceph-mon[25838]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:38.347 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:37 smithi033 ceph-mon[23750]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:40.162 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:39 smithi100 ceph-mon[25838]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:40.162 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:39 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:40.162 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:39 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:56:40.162 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:39 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:40.162 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:39 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:56:40.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:39 smithi033 ceph-mon[23750]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:40.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:39 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:40.218 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:39 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:56:40.218 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:39 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:40.218 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:39 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:56:41.142 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:56:41.160 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_4 2024-09-17T00:56:42.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:41 smithi033 ceph-mon[23750]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:42.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:41 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:42.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:41 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:42.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:41 smithi033 ceph-mon[23750]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T00:56:42.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:41 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:41 smithi100 ceph-mon[25838]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:41 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:41 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:41 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T00:56:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:41 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:43.125 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:42 smithi033 ceph-mon[23750]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:56:43.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:42 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:56:44.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:43 smithi033 ceph-mon[23750]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:43 smithi100 ceph-mon[25838]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:46.091 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:46.091 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:56:46.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:56:46.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:56:46.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:46.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:46.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:46.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:46.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:56:46.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:56:46.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='client.14236 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:56:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:56:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:56:46.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:47.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:46 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T00:56:47.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:46 smithi033 ceph-mon[23750]: Reconfiguring mon.smithi033 (unknown last config time)... 2024-09-17T00:56:47.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:46 smithi033 ceph-mon[23750]: Reconfiguring daemon mon.smithi033 on smithi033 2024-09-17T00:56:47.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:46 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T00:56:47.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:46 smithi100 ceph-mon[25838]: Reconfiguring mon.smithi033 (unknown last config time)... 2024-09-17T00:56:47.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:46 smithi100 ceph-mon[25838]: Reconfiguring daemon mon.smithi033 on smithi033 2024-09-17T00:56:48.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:47 smithi033 ceph-mon[23750]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:48.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:47 smithi100 ceph-mon[25838]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:49.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:49.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:48 smithi033 ceph-mon[23750]: Reconfiguring mgr.smithi033.illmog (unknown last config time)... 2024-09-17T00:56:49.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:56:49.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:56:49.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:49.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:48 smithi033 ceph-mon[23750]: Reconfiguring daemon mgr.smithi033.illmog on smithi033 2024-09-17T00:56:49.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:49.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:48 smithi100 ceph-mon[25838]: Reconfiguring mgr.smithi033.illmog (unknown last config time)... 2024-09-17T00:56:49.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:56:49.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:56:49.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:56:49.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:48 smithi100 ceph-mon[25838]: Reconfiguring daemon mgr.smithi033.illmog on smithi033 2024-09-17T00:56:50.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:49 smithi100 ceph-mon[25838]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:50.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:49 smithi033 ceph-mon[23750]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:51.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:50 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/36714338' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "52dcaf68-ea3d-48d0-9b79-e75911d2e73a"}]: dispatch 2024-09-17T00:56:51.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:50 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/36714338' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "52dcaf68-ea3d-48d0-9b79-e75911d2e73a"}]': finished 2024-09-17T00:56:51.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:50 smithi100 ceph-mon[25838]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T00:56:51.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:56:51.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:50 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2535631779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:56:51.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:50 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/36714338' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "52dcaf68-ea3d-48d0-9b79-e75911d2e73a"}]: dispatch 2024-09-17T00:56:51.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:50 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/36714338' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "52dcaf68-ea3d-48d0-9b79-e75911d2e73a"}]': finished 2024-09-17T00:56:51.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:50 smithi033 ceph-mon[23750]: osdmap e6: 1 total, 0 up, 1 in 2024-09-17T00:56:51.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:56:51.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:50 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2535631779' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:56:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:51 smithi100 ceph-mon[25838]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:52.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:51 smithi033 ceph-mon[23750]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:53.598 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:53 smithi033 ceph-mon[23750]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:53.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:53 smithi100 ceph-mon[25838]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:55.362 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:55 smithi033 ceph-mon[23750]: Reconfiguring alertmanager.smithi033 (dependencies changed)... 2024-09-17T00:56:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:55 smithi033 ceph-mon[23750]: Reconfiguring daemon alertmanager.smithi033 on smithi033 2024-09-17T00:56:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:55 smithi033 ceph-mon[23750]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:56:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:55 smithi100 ceph-mon[25838]: Reconfiguring alertmanager.smithi033 (dependencies changed)... 2024-09-17T00:56:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:55 smithi100 ceph-mon[25838]: Reconfiguring daemon alertmanager.smithi033 on smithi033 2024-09-17T00:56:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:55 smithi100 ceph-mon[25838]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:56:57 smithi033 ceph-mon[23750]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:56:58.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:56:57 smithi100 ceph-mon[25838]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:00.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:00 smithi033 ceph-mon[23750]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:00.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:00.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:00 smithi033 ceph-mon[23750]: Reconfiguring crash.smithi033 (monmap changed)... 2024-09-17T00:57:00.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:57:00.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:00.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:00 smithi033 ceph-mon[23750]: Reconfiguring daemon crash.smithi033 on smithi033 2024-09-17T00:57:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:00 smithi100 ceph-mon[25838]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:00 smithi100 ceph-mon[25838]: Reconfiguring crash.smithi033 (monmap changed)... 2024-09-17T00:57:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:57:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:00 smithi100 ceph-mon[25838]: Reconfiguring daemon crash.smithi033 on smithi033 2024-09-17T00:57:01.551 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T00:57:01.551 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:01.551 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:57:01.551 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:57:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T00:57:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:57:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:57:02.606 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:02 smithi033 ceph-mon[23750]: Deploying daemon osd.0 on smithi033 2024-09-17T00:57:02.606 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:02 smithi033 ceph-mon[23750]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:02.606 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:02 smithi100 ceph-mon[25838]: Deploying daemon osd.0 on smithi033 2024-09-17T00:57:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:02 smithi100 ceph-mon[25838]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:03.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:03 smithi033 ceph-mon[23750]: Reconfiguring grafana.smithi033 (dependencies changed)... 2024-09-17T00:57:03.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:03 smithi033 ceph-mon[23750]: Reconfiguring daemon grafana.smithi033 on smithi033 2024-09-17T00:57:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:03 smithi100 ceph-mon[25838]: Reconfiguring grafana.smithi033 (dependencies changed)... 2024-09-17T00:57:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:03 smithi100 ceph-mon[25838]: Reconfiguring daemon grafana.smithi033 on smithi033 2024-09-17T00:57:04.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:04 smithi033 ceph-mon[23750]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:04 smithi100 ceph-mon[25838]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:06.602 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:06 smithi033 ceph-mon[23750]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:06.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:06 smithi100 ceph-mon[25838]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:08.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:07 smithi033 ceph-mon[23750]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:08.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:08.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:07 smithi100 ceph-mon[25838]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:08.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:09.992 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:09 smithi033 ceph-mon[23750]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:09 smithi100 ceph-mon[25838]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:10.938 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:10 smithi033 ceph-mon[23750]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T00:57:11.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:10 smithi100 ceph-mon[25838]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T00:57:12.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:12.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T00:57:12.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T00:57:12.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:12.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:57:12.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:12.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: Reconfiguring crash.smithi100 (monmap changed)... 2024-09-17T00:57:12.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:57:12.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:12.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:11 smithi033 ceph-mon[23750]: Reconfiguring daemon crash.smithi100 on smithi100 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: osdmap e7: 1 total, 0 up, 1 in 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:12.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: Reconfiguring crash.smithi100 (monmap changed)... 2024-09-17T00:57:12.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:57:12.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:12.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:11 smithi100 ceph-mon[25838]: Reconfiguring daemon crash.smithi100 on smithi100 2024-09-17T00:57:12.303 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 0 on host 'smithi033' 2024-09-17T00:57:12.891 DEBUG:teuthology.orchestra.run.smithi033:osd.0> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.0.service 2024-09-17T00:57:12.894 INFO:tasks.cephadm:Deploying osd.1 on smithi033 with /dev/vg_nvme/lv_3... 2024-09-17T00:57:12.894 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T00:57:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:57:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T00:57:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:57:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:57:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: osdmap e8: 1 total, 0 up, 1 in 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:57:13.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: Reconfiguring mgr.smithi100.qrwkie (monmap changed)... 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: Reconfiguring daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696] boot 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T00:57:14.149 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:14.150 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:14.150 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:57:14.150 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:57:14.150 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:14.239 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:57:14.239 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:57:14.239 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: Reconfiguring mgr.smithi100.qrwkie (monmap changed)... 2024-09-17T00:57:14.239 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: Reconfiguring daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T00:57:14.239 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: osd.0 [v2:172.21.15.33:6802/2618793696,v1:172.21.15.33:6803/2618793696] boot 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: osdmap e9: 1 total, 1 up, 1 in 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:57:14.240 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: Reconfiguring mon.smithi100 (monmap changed)... 2024-09-17T00:57:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: Reconfiguring daemon mon.smithi100 on smithi100 2024-09-17T00:57:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T00:57:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.33:9093"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.33:3000"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.33:9095"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:15.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:15.245 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: Reconfiguring mon.smithi100 (monmap changed)... 2024-09-17T00:57:15.245 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: Reconfiguring daemon mon.smithi100 on smithi100 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: osdmap e10: 1 total, 1 up, 1 in 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.33:9093"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.33:3000"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.33:9095"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:15.246 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:15.247 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:15.459 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:57:15.481 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_3 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.33:9093"}]: dispatch 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.33:3000"}]: dispatch 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:57:16.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:15 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.33:9095"}]: dispatch 2024-09-17T00:57:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:57:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.33:9093"}]: dispatch 2024-09-17T00:57:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:57:16.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.33:3000"}]: dispatch 2024-09-17T00:57:16.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:57:16.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:15 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.33:9095"}]: dispatch 2024-09-17T00:57:17.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:17 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:17.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:17 smithi033 ceph-mon[23750]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:17 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:17 smithi100 ceph-mon[25838]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:18.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:18 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:57:18.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:18 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:57:18.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:18 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:18.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:18 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:57:18.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:18 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:57:18.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:18 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:19.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:19 smithi033 ceph-mon[23750]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:57:19.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:19 smithi033 ceph-mon[23750]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:19.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:19 smithi100 ceph-mon[25838]: from='client.14254 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:57:19.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:19 smithi100 ceph-mon[25838]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:21.967 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:21 smithi033 ceph-mon[23750]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:21.968 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:21 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:21 smithi100 ceph-mon[25838]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:21 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:23 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3137078464' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "53834228-46f9-4bc6-9a87-e1f3ceecef40"}]: dispatch 2024-09-17T00:57:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:23 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3137078464' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "53834228-46f9-4bc6-9a87-e1f3ceecef40"}]': finished 2024-09-17T00:57:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:23 smithi033 ceph-mon[23750]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T00:57:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:23 smithi033 ceph-mon[23750]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:23 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2446806559' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:57:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:23 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3137078464' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "53834228-46f9-4bc6-9a87-e1f3ceecef40"}]: dispatch 2024-09-17T00:57:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:23 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3137078464' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "53834228-46f9-4bc6-9a87-e1f3ceecef40"}]': finished 2024-09-17T00:57:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:23 smithi100 ceph-mon[25838]: osdmap e11: 2 total, 1 up, 2 in 2024-09-17T00:57:23.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:23.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:23 smithi100 ceph-mon[25838]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:23.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:23 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2446806559' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:57:26.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:25 smithi033 ceph-mon[23750]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:26.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:25 smithi100 ceph-mon[25838]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:27.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:57:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:29.205 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:28 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi033 2024-09-17T00:57:29.205 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:28 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi033 to 12868M 2024-09-17T00:57:29.236 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:28 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi033 2024-09-17T00:57:29.236 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:28 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi033 to 12868M 2024-09-17T00:57:29.981 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:29 smithi033 ceph-mon[23750]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:29.988 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:29 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:29 smithi100 ceph-mon[25838]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:29 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:31.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:30 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T00:57:31.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:30 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:31.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:30 smithi033 ceph-mon[23750]: Deploying daemon osd.1 on smithi033 2024-09-17T00:57:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:30 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T00:57:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:30 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:30 smithi100 ceph-mon[25838]: Deploying daemon osd.1 on smithi033 2024-09-17T00:57:31.985 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:31 smithi033 ceph-mon[23750]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:31 smithi100 ceph-mon[25838]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:33 smithi100 ceph-mon[25838]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:33.798 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:33 smithi033 ceph-mon[23750]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:36.521 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:36 smithi033 ceph-mon[23750]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:36.521 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:36 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:36.521 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:36 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:36.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:36 smithi100 ceph-mon[25838]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:36.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:36 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:36.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:36 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:37.589 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:37 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:37.590 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:37 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:37.590 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:37 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:37.590 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:37 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:37.718 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:37 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:37.718 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:37 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:37.719 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:37 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:37.719 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:37 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:38.458 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:38 smithi033 ceph-mon[23750]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:38.459 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:38 smithi100 ceph-mon[25838]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:40.338 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:40 smithi033 ceph-mon[23750]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:40.339 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:40 smithi033 ceph-mon[23750]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T00:57:40.350 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 1 on host 'smithi033' 2024-09-17T00:57:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:40 smithi100 ceph-mon[25838]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:40 smithi100 ceph-mon[25838]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T00:57:40.949 DEBUG:teuthology.orchestra.run.smithi033:osd.1> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.1.service 2024-09-17T00:57:40.951 INFO:tasks.cephadm:Deploying osd.2 on smithi033 with /dev/vg_nvme/lv_2... 2024-09-17T00:57:40.951 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T00:57:41.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-mon[23750]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T00:57:41.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-mon[23750]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T00:57:41.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:41.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-mon[23750]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:57:41.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:41.614 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[38252]: 2024-09-17T00:57:41.284+0000 7f58a2708700 -1 osd.1 0 waiting for initial osdmap 2024-09-17T00:57:41.614 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:57:41 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[38252]: 2024-09-17T00:57:41.287+0000 7f589c89e700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:57:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:41 smithi100 ceph-mon[25838]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T00:57:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:41 smithi100 ceph-mon[25838]: osdmap e12: 2 total, 1 up, 2 in 2024-09-17T00:57:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:41 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:41 smithi100 ceph-mon[25838]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:57:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:41 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:42 smithi033 ceph-mon[23750]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:42 smithi033 ceph-mon[23750]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:57:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:42 smithi033 ceph-mon[23750]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T00:57:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:42 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:42 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:42 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:42 smithi100 ceph-mon[25838]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:42 smithi100 ceph-mon[25838]: from='osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:57:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:42 smithi100 ceph-mon[25838]: osdmap e13: 2 total, 1 up, 2 in 2024-09-17T00:57:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:42 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:42 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:42 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:43.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:43 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:57:43.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:43 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:57:43.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:43 smithi033 ceph-mon[23750]: osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096] boot 2024-09-17T00:57:43.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:43 smithi033 ceph-mon[23750]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T00:57:43.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:43 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:43.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:43 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:57:43.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:43 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:57:43.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:43 smithi100 ceph-mon[25838]: osd.1 [v2:172.21.15.33:6810/1083858096,v1:172.21.15.33:6811/1083858096] boot 2024-09-17T00:57:43.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:43 smithi100 ceph-mon[25838]: osdmap e14: 2 total, 2 up, 2 in 2024-09-17T00:57:43.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:43 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:57:44.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:44 smithi033 ceph-mon[23750]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:44.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:44 smithi033 ceph-mon[23750]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T00:57:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:44 smithi100 ceph-mon[25838]: pgmap v53: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-17T00:57:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:44 smithi100 ceph-mon[25838]: osdmap e15: 2 total, 2 up, 2 in 2024-09-17T00:57:44.973 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:57:44.996 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_2 2024-09-17T00:57:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:46 smithi033 ceph-mon[23750]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:46.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:46 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:46 smithi100 ceph-mon[25838]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:46 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:47.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:47 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:57:47.328 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:47 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:57:47.328 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:47 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:47.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:47 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:57:47.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:47 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:57:47.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:47 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:48 smithi033 ceph-mon[23750]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:57:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:48 smithi033 ceph-mon[23750]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:48 smithi100 ceph-mon[25838]: from='client.14272 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:57:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:48 smithi100 ceph-mon[25838]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:57:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:57:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:50.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:50.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:50.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:57:50.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:50.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:57:51.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:51 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi033 to 6434M 2024-09-17T00:57:51.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:51 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/711747950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70c257f6-4971-449b-953b-4a301b9ed127"}]: dispatch 2024-09-17T00:57:51.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:51 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/711747950' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70c257f6-4971-449b-953b-4a301b9ed127"}]': finished 2024-09-17T00:57:51.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:51 smithi033 ceph-mon[23750]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T00:57:51.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:51 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:57:51.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:51 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi033 to 6434M 2024-09-17T00:57:51.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:51 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/711747950' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "70c257f6-4971-449b-953b-4a301b9ed127"}]: dispatch 2024-09-17T00:57:51.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:51 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/711747950' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "70c257f6-4971-449b-953b-4a301b9ed127"}]': finished 2024-09-17T00:57:51.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:51 smithi100 ceph-mon[25838]: osdmap e16: 3 total, 2 up, 3 in 2024-09-17T00:57:51.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:51 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:57:52.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:52 smithi033 ceph-mon[23750]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:52.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:52 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2333094363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:57:52.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:52 smithi100 ceph-mon[25838]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:52.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:52 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2333094363' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:57:53.609 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:53 smithi033 ceph-mon[23750]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:53.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:53 smithi100 ceph-mon[25838]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:55.715 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:55.716 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:55.716 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:55 smithi033 ceph-mon[23750]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:57:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:55 smithi100 ceph-mon[25838]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:58.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:57 smithi033 ceph-mon[23750]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:58.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:57 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T00:57:58.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:57 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:58.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:57 smithi033 ceph-mon[23750]: Deploying daemon osd.2 on smithi033 2024-09-17T00:57:58.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:57 smithi100 ceph-mon[25838]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:57:58.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:57 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T00:57:58.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:57 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:57:58.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:57 smithi100 ceph-mon[25838]: Deploying daemon osd.2 on smithi033 2024-09-17T00:58:00.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:57:59 smithi033 ceph-mon[23750]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:00.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:57:59 smithi100 ceph-mon[25838]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:01.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:58:01.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:58:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:58:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:58:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:01 smithi033 ceph-mon[23750]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:02.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:01 smithi100 ceph-mon[25838]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:03.203 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:03.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:03.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:03.204 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:04.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:03 smithi033 ceph-mon[23750]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:04.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:03 smithi100 ceph-mon[25838]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:05.441 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 2 on host 'smithi033' 2024-09-17T00:58:06.014 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:05 smithi033 ceph-mon[23750]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:06.014 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:05 smithi033 ceph-mon[23750]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T00:58:06.014 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:05 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:06.041 DEBUG:teuthology.orchestra.run.smithi033:osd.2> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.2.service 2024-09-17T00:58:06.043 INFO:tasks.cephadm:Deploying osd.3 on smithi033 with /dev/vg_nvme/lv_1... 2024-09-17T00:58:06.043 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T00:58:06.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:05 smithi100 ceph-mon[25838]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:06.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:05 smithi100 ceph-mon[25838]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T00:58:06.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:05 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:07.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:06 smithi100 ceph-mon[25838]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T00:58:07.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:06 smithi100 ceph-mon[25838]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T00:58:07.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:06 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:07.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:06 smithi100 ceph-mon[25838]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:58:07.254 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:06 smithi033 ceph-mon[23750]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T00:58:07.254 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:06 smithi033 ceph-mon[23750]: osdmap e17: 3 total, 2 up, 3 in 2024-09-17T00:58:07.254 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:06 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:07.254 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:06 smithi033 ceph-mon[23750]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:58:07.254 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:06 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[41913]: 2024-09-17T00:58:06.970+0000 7f36d6d67700 -1 osd.2 0 waiting for initial osdmap 2024-09-17T00:58:07.255 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:06 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[41913]: 2024-09-17T00:58:06.976+0000 7f36d16fe700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:58:08.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:07 smithi033 ceph-mon[23750]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:08.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:07 smithi033 ceph-mon[23750]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:58:08.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:07 smithi033 ceph-mon[23750]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T00:58:08.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:08.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:07 smithi100 ceph-mon[25838]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:07 smithi100 ceph-mon[25838]: from='osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:58:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:07 smithi100 ceph-mon[25838]: osdmap e18: 3 total, 2 up, 3 in 2024-09-17T00:58:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:09 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:58:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:09 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:58:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:09 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:09 smithi033 ceph-mon[23750]: osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113] boot 2024-09-17T00:58:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:09 smithi033 ceph-mon[23750]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T00:58:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:09 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:09.660 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T00:58:09.680 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi033:vg_nvme/lv_1 2024-09-17T00:58:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:09 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:58:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:09 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:58:09.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:09 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:09.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:09 smithi100 ceph-mon[25838]: osd.2 [v2:172.21.15.33:6818/255012113,v1:172.21.15.33:6819/255012113] boot 2024-09-17T00:58:09.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:09 smithi100 ceph-mon[25838]: osdmap e19: 3 total, 3 up, 3 in 2024-09-17T00:58:09.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:09 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:58:10.535 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:10 smithi033 ceph-mon[23750]: pgmap v71: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:10.535 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T00:58:10.535 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T00:58:10.535 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:10 smithi033 ceph-mon[23750]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T00:58:10.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T00:58:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:10 smithi100 ceph-mon[25838]: pgmap v71: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-17T00:58:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-17T00:58:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-17T00:58:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:10 smithi100 ceph-mon[25838]: osdmap e20: 3 total, 3 up, 3 in 2024-09-17T00:58:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-17T00:58:11.452 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T00:58:11.452 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:11 smithi033 ceph-mon[23750]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T00:58:11.452 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:11.452 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:11 smithi033 ceph-mon[23750]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-17T00:58:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:11 smithi100 ceph-mon[25838]: osdmap e21: 3 total, 3 up, 3 in 2024-09-17T00:58:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:11 smithi100 ceph-mon[25838]: pgmap v74: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:11.728 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 00:58:11 smithi033 sudo[43398]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:58:11.728 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 00:58:11 smithi033 sudo[43398]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:11.728 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 00:58:11 smithi033 sudo[43398]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:12.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 00:58:11 smithi033 sudo[43421]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:58:12.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 00:58:11 smithi033 sudo[43421]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:12.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 00:58:11 smithi033 sudo[43421]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:12.582 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:58:12 smithi033 sudo[43426]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:58:12.582 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:58:12 smithi033 sudo[43426]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:12.582 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:58:12 smithi033 sudo[43426]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:12.841 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:12 smithi033 ceph-mon[23750]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T00:58:12.841 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:12 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:13.113 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:58:12 smithi033 sudo[43455]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:58:13.113 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:58:12 smithi033 sudo[43455]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:13.113 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 00:58:12 smithi033 sudo[43455]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:13.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:12 smithi100 ceph-mon[25838]: osdmap e22: 3 total, 3 up, 3 in 2024-09-17T00:58:13.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:12 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:13.486 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:13 smithi033 sudo[43526]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:58:13.486 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:13 smithi033 sudo[43526]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:13.486 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:13 smithi033 sudo[43526]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:14.113 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:13 smithi033 sudo[43610]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:58:14.113 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:13 smithi033 sudo[43610]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:14.113 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 00:58:13 smithi033 sudo[43610]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:14.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 sudo[43627]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:58:14.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 sudo[43627]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:14.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 sudo[43627]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 sudo[43632]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 sudo[43632]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 sudo[43632]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:58:14.756 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:14.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: mgrmap e19: smithi033.illmog(active, since 2m), standbys: smithi100.qrwkie 2024-09-17T00:58:14.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:14 smithi033 ceph-mon[23750]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: pgmap v76: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: from='client.14294 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: mgrmap e19: smithi033.illmog(active, since 2m), standbys: smithi100.qrwkie 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 ceph-mon[25838]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 sudo[28155]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 sudo[28155]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:14.907 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:14 smithi100 sudo[28155]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 sudo[28158]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:58:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 sudo[28158]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:58:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 sudo[28158]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:58:15.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi033 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:15.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi033 to 4289M 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:15.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:15 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:58:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:58:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:58:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:58:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T00:58:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi033 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi033 to 4289M 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:15.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:15.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:15 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:16 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3341251733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cdd9da87-c04b-4213-a53c-39823c494cfa"}]: dispatch 2024-09-17T00:58:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:16 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3341251733' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cdd9da87-c04b-4213-a53c-39823c494cfa"}]': finished 2024-09-17T00:58:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:16 smithi100 ceph-mon[25838]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T00:58:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:16 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:17.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:16 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3341251733' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cdd9da87-c04b-4213-a53c-39823c494cfa"}]: dispatch 2024-09-17T00:58:17.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:16 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3341251733' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cdd9da87-c04b-4213-a53c-39823c494cfa"}]': finished 2024-09-17T00:58:17.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:16 smithi033 ceph-mon[23750]: osdmap e23: 4 total, 3 up, 4 in 2024-09-17T00:58:17.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:16 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:17 smithi100 ceph-mon[25838]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:17 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1962687452' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:58:18.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:17 smithi033 ceph-mon[23750]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:18.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:17 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1962687452' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:58:20.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:19 smithi033 ceph-mon[23750]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-17T00:58:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:19 smithi100 ceph-mon[25838]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-17T00:58:22.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:21 smithi033 ceph-mon[23750]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T00:58:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:21 smithi100 ceph-mon[25838]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-17T00:58:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T00:58:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:23 smithi100 ceph-mon[25838]: Deploying daemon osd.3 on smithi033 2024-09-17T00:58:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:23 smithi100 ceph-mon[25838]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T00:58:23.494 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T00:58:23.494 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:23.494 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:23 smithi033 ceph-mon[23750]: Deploying daemon osd.3 on smithi033 2024-09-17T00:58:23.494 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:23 smithi033 ceph-mon[23750]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-17T00:58:26.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:25 smithi033 ceph-mon[23750]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T00:58:26.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:25 smithi100 ceph-mon[25838]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-17T00:58:27.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:27 smithi100 ceph-mon[25838]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2024-09-17T00:58:27.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:27.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:27.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:27.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:27 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:27 smithi033 ceph-mon[23750]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.7 KiB/s rd, 74 KiB/s wr, 5 op/s 2024-09-17T00:58:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:27 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:30.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:29 smithi033 ceph-mon[23750]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T00:58:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:29 smithi100 ceph-mon[25838]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-17T00:58:30.396 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 3 on host 'smithi033' 2024-09-17T00:58:31.057 DEBUG:teuthology.orchestra.run.smithi033:osd.3> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.3.service 2024-09-17T00:58:31.060 INFO:tasks.cephadm:Deploying osd.4 on smithi100 with /dev/vg_nvme/lv_4... 2024-09-17T00:58:31.060 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-17T00:58:31.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:31 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:31.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:31 smithi033 ceph-mon[23750]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T00:58:31.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:31 smithi033 ceph-mon[23750]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:31.694 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:31 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:31.694 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:31 smithi100 ceph-mon[25838]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T00:58:31.694 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:31 smithi100 ceph-mon[25838]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:32.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:32 smithi100 ceph-mon[25838]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T00:58:32.553 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:32 smithi100 ceph-mon[25838]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T00:58:32.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:32 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:32.554 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:32 smithi100 ceph-mon[25838]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:58:32.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:32 smithi033 ceph-mon[23750]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T00:58:32.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:32 smithi033 ceph-mon[23750]: osdmap e24: 4 total, 3 up, 4 in 2024-09-17T00:58:32.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:32 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:32.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:32 smithi033 ceph-mon[23750]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T00:58:32.678 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 00:58:32 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[45132]: 2024-09-17T00:58:32.410+0000 7f86f0b96700 -1 osd.3 0 waiting for initial osdmap 2024-09-17T00:58:32.678 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 00:58:32 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[45132]: 2024-09-17T00:58:32.415+0000 7f86ecd30700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:58:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:33 smithi100 ceph-mon[25838]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:58:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:33 smithi100 ceph-mon[25838]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T00:58:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:33 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:33 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:33 smithi100 ceph-mon[25838]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:33 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:33.817 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:58:33.835 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_4 2024-09-17T00:58:33.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:33 smithi033 ceph-mon[23750]: from='osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-17T00:58:33.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:33 smithi033 ceph-mon[23750]: osdmap e25: 4 total, 3 up, 4 in 2024-09-17T00:58:33.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:33 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:33.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:33 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:33.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:33 smithi033 ceph-mon[23750]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2024-09-17T00:58:33.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:33 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:34 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:58:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:34 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:58:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:34 smithi033 ceph-mon[23750]: osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389] boot 2024-09-17T00:58:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:34 smithi033 ceph-mon[23750]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T00:58:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:34 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:34 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:34.676 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:34 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:58:34.676 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:34 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:58:34.676 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:34 smithi100 ceph-mon[25838]: osd.3 [v2:172.21.15.33:6826/1776786389,v1:172.21.15.33:6827/1776786389] boot 2024-09-17T00:58:34.676 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:34 smithi100 ceph-mon[25838]: osdmap e26: 4 total, 4 up, 4 in 2024-09-17T00:58:34.676 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:34 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:58:34.676 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:34 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:35 smithi100 ceph-mon[25838]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T00:58:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:35 smithi100 ceph-mon[25838]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:35.720 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:35 smithi033 ceph-mon[23750]: osdmap e27: 4 total, 4 up, 4 in 2024-09-17T00:58:35.720 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:35 smithi033 ceph-mon[23750]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:36.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:36 smithi100 ceph-mon[25838]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:58:36.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:36 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:58:36.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:36 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:58:36.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:36 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:36.802 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:36 smithi033 ceph-mon[23750]: from='client.24151 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:58:36.802 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:36 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:58:36.802 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:36 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:58:36.802 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:36 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:37.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:37 smithi100 ceph-mon[25838]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:37.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:37 smithi033 ceph-mon[23750]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:38.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi033 2024-09-17T00:58:38.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi033 to 3217M 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/599643594' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4848def-64f9-43f2-a0e3-0817e553032d"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4848def-64f9-43f2-a0e3-0817e553032d"}]: dispatch 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a4848def-64f9-43f2-a0e3-0817e553032d"}]': finished 2024-09-17T00:58:38.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T00:58:38.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:38.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:38 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/99468751' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi033 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi033 to 3217M 2024-09-17T00:58:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/599643594' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4848def-64f9-43f2-a0e3-0817e553032d"}]: dispatch 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a4848def-64f9-43f2-a0e3-0817e553032d"}]: dispatch 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a4848def-64f9-43f2-a0e3-0817e553032d"}]': finished 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: osdmap e28: 5 total, 4 up, 5 in 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:38.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:38 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/99468751' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:58:39.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:39 smithi100 ceph-mon[25838]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:39.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:39 smithi033 ceph-mon[23750]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:42.079 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:41 smithi100 ceph-mon[25838]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:41 smithi033 ceph-mon[23750]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:43.689 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:43 smithi100 ceph-mon[25838]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:43.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:43 smithi033 ceph-mon[23750]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:44.710 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:44 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T00:58:44.710 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:44 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:44.710 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:44 smithi100 ceph-mon[25838]: Deploying daemon osd.4 on smithi100 2024-09-17T00:58:44.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:44 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T00:58:44.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:44 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:44.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:44 smithi033 ceph-mon[23750]: Deploying daemon osd.4 on smithi100 2024-09-17T00:58:45.616 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:45 smithi100 ceph-mon[25838]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:45.862 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:45 smithi033 ceph-mon[23750]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:48.084 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:47 smithi100 ceph-mon[25838]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:48.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:47 smithi033 ceph-mon[23750]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:50.196 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:49 smithi100 ceph-mon[25838]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:50.197 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:50.197 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:50.197 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:50.197 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:49 smithi033 ceph-mon[23750]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:58:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:50.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:58:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:51 smithi100 ceph-mon[25838]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:52.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:51 smithi033 ceph-mon[23750]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:52.389 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 4 on host 'smithi100' 2024-09-17T00:58:52.968 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:52 smithi100 ceph-mon[25838]: from='osd.4 [v2:172.21.15.100:6800/588543990,v1:172.21.15.100:6801/588543990]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:58:52.968 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:52 smithi100 ceph-mon[25838]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:58:52.969 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:52 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:52.992 DEBUG:teuthology.orchestra.run.smithi100:osd.4> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4.service 2024-09-17T00:58:52.994 INFO:tasks.cephadm:Deploying osd.5 on smithi100 with /dev/vg_nvme/lv_3... 2024-09-17T00:58:52.994 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-17T00:58:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:52 smithi033 ceph-mon[23750]: from='osd.4 [v2:172.21.15.100:6800/588543990,v1:172.21.15.100:6801/588543990]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:58:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:52 smithi033 ceph-mon[23750]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:58:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:52 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-mon[25838]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-mon[25838]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T00:58:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-mon[25838]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T00:58:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-mon[25838]: from='osd.4 [v2:172.21.15.100:6800/588543990,v1:172.21.15.100:6801/588543990]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:58:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-mon[25838]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:58:54.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[30053]: 2024-09-17T00:58:53.965+0000 7fe1bad49700 -1 osd.4 0 waiting for initial osdmap 2024-09-17T00:58:54.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 00:58:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[30053]: 2024-09-17T00:58:53.969+0000 7fe1b4edf700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:58:54.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:53 smithi033 ceph-mon[23750]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:54.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:53 smithi033 ceph-mon[23750]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T00:58:54.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:53 smithi033 ceph-mon[23750]: osdmap e29: 5 total, 4 up, 5 in 2024-09-17T00:58:54.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:53 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:54.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:53 smithi033 ceph-mon[23750]: from='osd.4 [v2:172.21.15.100:6800/588543990,v1:172.21.15.100:6801/588543990]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:58:54.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:53 smithi033 ceph-mon[23750]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:58:55.084 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:54 smithi100 ceph-mon[25838]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T00:58:55.084 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:54 smithi100 ceph-mon[25838]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T00:58:55.084 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:55.084 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:55.084 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:55.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:54 smithi033 ceph-mon[23750]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T00:58:55.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:54 smithi033 ceph-mon[23750]: osdmap e30: 5 total, 4 up, 5 in 2024-09-17T00:58:55.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:55.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:55.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:58:56.160 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:58:56.179 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_3 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: osd.4 [v2:172.21.15.100:6800/588543990,v1:172.21.15.100:6801/588543990] boot 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T00:58:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: osd.4 [v2:172.21.15.100:6800/588543990,v1:172.21.15.100:6801/588543990] boot 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: osdmap e31: 5 total, 5 up, 5 in 2024-09-17T00:58:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:58:57.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:56 smithi100 ceph-mon[25838]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T00:58:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:56 smithi033 ceph-mon[23750]: osdmap e32: 5 total, 5 up, 5 in 2024-09-17T00:58:58.176 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:57 smithi100 ceph-mon[25838]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:58:58.176 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:57 smithi100 ceph-mon[25838]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T00:58:58.176 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:57 smithi100 ceph-mon[25838]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T00:58:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:57 smithi033 ceph-mon[23750]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:58:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:57 smithi033 ceph-mon[23750]: osdmap e33: 5 total, 5 up, 5 in 2024-09-17T00:58:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:57 smithi033 ceph-mon[23750]: osdmap e34: 5 total, 5 up, 5 in 2024-09-17T00:58:59.102 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:58 smithi100 ceph-mon[25838]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:58:59.103 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:58 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:58:59.103 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:58 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:58:59.103 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:58 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:59.103 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:58 smithi100 ceph-mon[25838]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T00:58:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:58 smithi033 ceph-mon[23750]: from='client.24169 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:58:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:58 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:58:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:58 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:58:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:58 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:58:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:58 smithi033 ceph-mon[23750]: osdmap e35: 5 total, 5 up, 5 in 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:00.226 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:58:59 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: pgmap v112: 1 pgs: 1 peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail; 133 KiB/s, 0 objects/s recovering 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:00.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:58:59 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:00 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi100 2024-09-17T00:59:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:00 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi100 to 15935M 2024-09-17T00:59:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:59:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:59:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:00 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi100 2024-09-17T00:59:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:00 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi100 to 15935M 2024-09-17T00:59:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T00:59:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/382201169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99c0933a-634c-4d6c-8bd4-dcd99e8121de"}]: dispatch 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99c0933a-634c-4d6c-8bd4-dcd99e8121de"}]: dispatch 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99c0933a-634c-4d6c-8bd4-dcd99e8121de"}]': finished 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:01 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/1200560053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 106 KiB/s, 0 objects/s recovering 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/382201169' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99c0933a-634c-4d6c-8bd4-dcd99e8121de"}]: dispatch 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99c0933a-634c-4d6c-8bd4-dcd99e8121de"}]: dispatch 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99c0933a-634c-4d6c-8bd4-dcd99e8121de"}]': finished 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: osdmap e36: 6 total, 5 up, 6 in 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:01 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/1200560053' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:59:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:03 smithi100 ceph-mon[25838]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-09-17T00:59:03.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:03 smithi033 ceph-mon[23750]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 88 KiB/s, 0 objects/s recovering 2024-09-17T00:59:05.961 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:05 smithi100 ceph-mon[25838]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-17T00:59:06.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:05 smithi033 ceph-mon[23750]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-17T00:59:08.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:07 smithi100 ceph-mon[25838]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:08.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T00:59:08.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:08.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:07 smithi033 ceph-mon[23750]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:08.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T00:59:08.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:09.136 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:08 smithi100 ceph-mon[25838]: Deploying daemon osd.5 on smithi100 2024-09-17T00:59:09.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:08 smithi033 ceph-mon[23750]: Deploying daemon osd.5 on smithi100 2024-09-17T00:59:10.088 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:09 smithi100 ceph-mon[25838]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:10.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:09 smithi033 ceph-mon[23750]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:12.021 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:11 smithi100 ceph-mon[25838]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:11 smithi033 ceph-mon[23750]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:13 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:13 smithi033 ceph-mon[23750]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:13.914 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:13.914 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:13.914 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:13.914 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:13 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:13.914 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:13 smithi100 ceph-mon[25838]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:15.964 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 5 on host 'smithi100' 2024-09-17T00:59:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:15 smithi100 ceph-mon[25838]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:15 smithi100 ceph-mon[25838]: from='osd.5 [v2:172.21.15.100:6808/2921136856,v1:172.21.15.100:6809/2921136856]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:59:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:15 smithi100 ceph-mon[25838]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:59:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:15 smithi033 ceph-mon[23750]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:15 smithi033 ceph-mon[23750]: from='osd.5 [v2:172.21.15.100:6808/2921136856,v1:172.21.15.100:6809/2921136856]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:59:16.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:15 smithi033 ceph-mon[23750]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:59:16.662 DEBUG:teuthology.orchestra.run.smithi100:osd.5> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.5.service 2024-09-17T00:59:16.665 INFO:tasks.cephadm:Deploying osd.6 on smithi100 with /dev/vg_nvme/lv_2... 2024-09-17T00:59:16.665 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-17T00:59:17.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-mon[25838]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T00:59:17.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-mon[25838]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T00:59:17.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:17.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-mon[25838]: from='osd.5 [v2:172.21.15.100:6808/2921136856,v1:172.21.15.100:6809/2921136856]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:17.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-mon[25838]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:17.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:17.183 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[33011]: 2024-09-17T00:59:16.905+0000 7fd972f65700 -1 osd.5 0 waiting for initial osdmap 2024-09-17T00:59:17.183 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 00:59:16 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[33011]: 2024-09-17T00:59:16.910+0000 7fd96b8f8700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:59:17.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:16 smithi033 ceph-mon[23750]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T00:59:17.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:16 smithi033 ceph-mon[23750]: osdmap e37: 6 total, 5 up, 6 in 2024-09-17T00:59:17.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:16 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:17.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:16 smithi033 ceph-mon[23750]: from='osd.5 [v2:172.21.15.100:6808/2921136856,v1:172.21.15.100:6809/2921136856]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:17.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:16 smithi033 ceph-mon[23750]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:17.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:16 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: osd.5 [v2:172.21.15.100:6808/2921136856,v1:172.21.15.100:6809/2921136856] boot 2024-09-17T00:59:18.181 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T00:59:18.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:18.182 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:17 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: osdmap e38: 6 total, 5 up, 6 in 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: osd.5 [v2:172.21.15.100:6808/2921136856,v1:172.21.15.100:6809/2921136856] boot 2024-09-17T00:59:18.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: osdmap e39: 6 total, 6 up, 6 in 2024-09-17T00:59:18.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:59:18.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:17 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:19.638 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:19 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:59:19.639 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:19 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:59:19.639 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:19 smithi100 ceph-mon[25838]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T00:59:19.639 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:19 smithi100 ceph-mon[25838]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:19.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:19 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:59:19.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:19 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:59:19.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:19 smithi033 ceph-mon[23750]: osdmap e40: 6 total, 6 up, 6 in 2024-09-17T00:59:19.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:19 smithi033 ceph-mon[23750]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:20 smithi100 ceph-mon[25838]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T00:59:20.509 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:59:20.531 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_2 2024-09-17T00:59:20.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:20 smithi033 ceph-mon[23750]: osdmap e41: 6 total, 6 up, 6 in 2024-09-17T00:59:21.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:21 smithi100 ceph-mon[25838]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T00:59:21.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:21 smithi100 ceph-mon[25838]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:21 smithi033 ceph-mon[23750]: osdmap e42: 6 total, 6 up, 6 in 2024-09-17T00:59:21.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:21 smithi033 ceph-mon[23750]: pgmap v130: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:22.725 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:22 smithi100 ceph-mon[25838]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T00:59:22.725 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:22 smithi100 ceph-mon[25838]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T00:59:22.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:22 smithi033 ceph-mon[23750]: osdmap e43: 6 total, 6 up, 6 in 2024-09-17T00:59:22.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:22 smithi033 ceph-mon[23750]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T00:59:24.074 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:24.074 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi100 2024-09-17T00:59:24.074 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.074 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi100 to 7967M 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:59:24.075 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:59:24.076 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:24.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:24.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi100 2024-09-17T00:59:24.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:24.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi100 to 7967M 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:59:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:25.223 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:25 smithi100 ceph-mon[25838]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:59:25.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:25 smithi033 ceph-mon[23750]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: Cluster is now healthy 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/3581475727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a78e41af-3b2b-4546-b469-15276fce2c91"}]: dispatch 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a78e41af-3b2b-4546-b469-15276fce2c91"}]: dispatch 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a78e41af-3b2b-4546-b469-15276fce2c91"}]': finished 2024-09-17T00:59:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T00:59:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:26 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T00:59:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: Cluster is now healthy 2024-09-17T00:59:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/3581475727' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a78e41af-3b2b-4546-b469-15276fce2c91"}]: dispatch 2024-09-17T00:59:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a78e41af-3b2b-4546-b469-15276fce2c91"}]: dispatch 2024-09-17T00:59:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a78e41af-3b2b-4546-b469-15276fce2c91"}]': finished 2024-09-17T00:59:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: osdmap e44: 7 total, 6 up, 7 in 2024-09-17T00:59:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:26 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:27.328 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:27 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/138949167' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:59:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:27 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/138949167' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:59:28.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:28 smithi033 ceph-mon[23750]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-17T00:59:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:28 smithi100 ceph-mon[25838]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-09-17T00:59:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:30 smithi100 ceph-mon[25838]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T00:59:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:30 smithi033 ceph-mon[23750]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-17T00:59:32.210 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:32 smithi100 ceph-mon[25838]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-17T00:59:32.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:32 smithi033 ceph-mon[23750]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-17T00:59:33.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:33 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T00:59:33.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:33 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:33.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:33 smithi100 ceph-mon[25838]: Deploying daemon osd.6 on smithi100 2024-09-17T00:59:33.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:33 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T00:59:33.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:33 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:33.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:33 smithi033 ceph-mon[23750]: Deploying daemon osd.6 on smithi100 2024-09-17T00:59:34.223 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:34 smithi100 ceph-mon[25838]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T00:59:34.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:34 smithi033 ceph-mon[23750]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T00:59:36.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:36 smithi033 ceph-mon[23750]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:36.456 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:36 smithi100 ceph-mon[25838]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:38 smithi033 ceph-mon[23750]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:38.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:38 smithi100 ceph-mon[25838]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:38.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:38.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:38.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:38.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:40.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:40 smithi033 ceph-mon[23750]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:40.430 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:40 smithi100 ceph-mon[25838]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:41.131 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:41 smithi100 ceph-mon[25838]: from='osd.6 [v2:172.21.15.100:6816/1888615208,v1:172.21.15.100:6817/1888615208]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:59:41.131 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:41 smithi100 ceph-mon[25838]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:59:41.162 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 6 on host 'smithi100' 2024-09-17T00:59:41.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:41 smithi033 ceph-mon[23750]: from='osd.6 [v2:172.21.15.100:6816/1888615208,v1:172.21.15.100:6817/1888615208]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:59:41.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:41 smithi033 ceph-mon[23750]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:59:41.852 DEBUG:teuthology.orchestra.run.smithi100:osd.6> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.6.service 2024-09-17T00:59:41.854 INFO:tasks.cephadm:Deploying osd.7 on smithi100 with /dev/vg_nvme/lv_1... 2024-09-17T00:59:41.855 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-17T00:59:42.203 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[35977]: 2024-09-17T00:59:42.056+0000 7f69e624e700 -1 osd.6 0 waiting for initial osdmap 2024-09-17T00:59:42.203 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[35977]: 2024-09-17T00:59:42.066+0000 7f69e0be5700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:59:42.203 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:42.203 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T00:59:42.203 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T00:59:42.204 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:42.204 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: from='osd.6 [v2:172.21.15.100:6816/1888615208,v1:172.21.15.100:6817/1888615208]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:42.204 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:42.204 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:42.204 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:42 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T00:59:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: osdmap e45: 7 total, 6 up, 7 in 2024-09-17T00:59:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: from='osd.6 [v2:172.21.15.100:6816/1888615208,v1:172.21.15.100:6817/1888615208]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T00:59:42.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:42.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:42 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:43.309 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T00:59:43.309 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T00:59:43.309 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:43.309 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:43.309 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: osd.6 [v2:172.21.15.100:6816/1888615208,v1:172.21.15.100:6817/1888615208] boot 2024-09-17T00:59:43.309 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T00:59:43.310 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:43 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: osdmap e46: 7 total, 6 up, 7 in 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: osd.6 [v2:172.21.15.100:6816/1888615208,v1:172.21.15.100:6817/1888615208] boot 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: osdmap e47: 7 total, 7 up, 7 in 2024-09-17T00:59:43.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:43 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:59:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:44 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T00:59:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:44 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T00:59:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:44 smithi100 ceph-mon[25838]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:44 smithi100 ceph-mon[25838]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T00:59:44.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:44 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T00:59:44.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:44 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T00:59:44.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:44 smithi033 ceph-mon[23750]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-17T00:59:44.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:44 smithi033 ceph-mon[23750]: osdmap e48: 7 total, 7 up, 7 in 2024-09-17T00:59:45.479 INFO:teuthology.orchestra.run.smithi100.stdout: 2024-09-17T00:59:45.501 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi100:vg_nvme/lv_1 2024-09-17T00:59:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:45 smithi100 ceph-mon[25838]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T00:59:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:45 smithi100 ceph-mon[25838]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T00:59:45.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:45 smithi033 ceph-mon[23750]: osdmap e49: 7 total, 7 up, 7 in 2024-09-17T00:59:45.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:45 smithi033 ceph-mon[23750]: pgmap v149: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-17T00:59:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:46 smithi100 ceph-mon[25838]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T00:59:46.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:46 smithi033 ceph-mon[23750]: osdmap e50: 7 total, 7 up, 7 in 2024-09-17T00:59:47.714 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:47 smithi100 ceph-mon[25838]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T00:59:47.714 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:47 smithi100 ceph-mon[25838]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T00:59:47.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:47 smithi033 ceph-mon[23750]: osdmap e51: 7 total, 7 up, 7 in 2024-09-17T00:59:47.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:47 smithi033 ceph-mon[23750]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T00:59:48.579 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi100 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:48.737 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:48.738 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:48.738 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='client.24205 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi100:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi100 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:59:48.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:48.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:59:48.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:48.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:59:48.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T00:59:49.647 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:49 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi100 to 5311M 2024-09-17T00:59:49.647 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:49 smithi100 ceph-mon[25838]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-17T00:59:49.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:49 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi100 to 5311M 2024-09-17T00:59:49.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:49 smithi033 ceph-mon[23750]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-17T00:59:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:50 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/1904404087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b427a09-c6cf-4a16-9f36-44f98f7605a7"}]: dispatch 2024-09-17T00:59:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:50 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b427a09-c6cf-4a16-9f36-44f98f7605a7"}]: dispatch 2024-09-17T00:59:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:50 smithi100 ceph-mon[25838]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b427a09-c6cf-4a16-9f36-44f98f7605a7"}]': finished 2024-09-17T00:59:50.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:50 smithi100 ceph-mon[25838]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T00:59:50.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:50 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:59:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:50 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/1904404087' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b427a09-c6cf-4a16-9f36-44f98f7605a7"}]: dispatch 2024-09-17T00:59:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:50 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7b427a09-c6cf-4a16-9f36-44f98f7605a7"}]: dispatch 2024-09-17T00:59:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:50 smithi033 ceph-mon[23750]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7b427a09-c6cf-4a16-9f36-44f98f7605a7"}]': finished 2024-09-17T00:59:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:50 smithi033 ceph-mon[23750]: osdmap e52: 8 total, 7 up, 8 in 2024-09-17T00:59:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:50 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:59:51.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:51 smithi100 ceph-mon[25838]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T00:59:51.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:51 smithi100 ceph-mon[25838]: from='client.? 172.21.15.100:0/2601998209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:59:51.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:51 smithi033 ceph-mon[23750]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T00:59:51.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:51 smithi033 ceph-mon[23750]: from='client.? 172.21.15.100:0/2601998209' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-17T00:59:53.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:53 smithi100 ceph-mon[25838]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-17T00:59:53.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:53 smithi033 ceph-mon[23750]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-17T00:59:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:55 smithi100 ceph-mon[25838]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-17T00:59:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:55 smithi033 ceph-mon[23750]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-17T00:59:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:56 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T00:59:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:56 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:56 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T00:59:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:56 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:59:57.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:57 smithi100 ceph-mon[25838]: Deploying daemon osd.7 on smithi100 2024-09-17T00:59:57.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:57 smithi100 ceph-mon[25838]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T00:59:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:57 smithi033 ceph-mon[23750]: Deploying daemon osd.7 on smithi100 2024-09-17T00:59:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:57 smithi033 ceph-mon[23750]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T00:59:59.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 00:59:59 smithi100 ceph-mon[25838]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 00:59:59 smithi033 ceph-mon[23750]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:00 smithi100 ceph-mon[25838]: overall HEALTH_OK 2024-09-17T01:00:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:00:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:00:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:00 smithi033 ceph-mon[23750]: overall HEALTH_OK 2024-09-17T01:00:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:00:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:00:02.029 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:01 smithi100 ceph-mon[25838]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:02.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:01 smithi033 ceph-mon[23750]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:03.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:03 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:03.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:03 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:00:03.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:03 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:00:03.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:03 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:00:03.370 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:03 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:03.370 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:03 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:00:03.370 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:03 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:00:03.370 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:03 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:00:04.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:04 smithi033 ceph-mon[23750]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:04 smithi100 ceph-mon[25838]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:05.188 INFO:teuthology.orchestra.run.smithi100.stdout:Created osd(s) 7 on host 'smithi100' 2024-09-17T01:00:05.866 DEBUG:teuthology.orchestra.run.smithi100:osd.7> sudo journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.7.service 2024-09-17T01:00:05.868 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-17T01:00:05.868 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-17T01:00:06.108 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:06 smithi100 ceph-mon[25838]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:06.108 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:06 smithi100 ceph-mon[25838]: from='osd.7 [v2:172.21.15.100:6824/3730782588,v1:172.21.15.100:6825/3730782588]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:00:06.108 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:06 smithi100 ceph-mon[25838]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:00:06.108 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:06 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:06.195 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:06 smithi033 ceph-mon[23750]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:06.195 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:06 smithi033 ceph-mon[23750]: from='osd.7 [v2:172.21.15.100:6824/3730782588,v1:172.21.15.100:6825/3730782588]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:00:06.195 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:06 smithi033 ceph-mon[23750]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:00:06.195 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:06 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:07.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:07 smithi033 ceph-mon[23750]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:00:07.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:07 smithi033 ceph-mon[23750]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T01:00:07.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:07.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:07 smithi033 ceph-mon[23750]: from='osd.7 [v2:172.21.15.100:6824/3730782588,v1:172.21.15.100:6825/3730782588]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:00:07.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:07 smithi033 ceph-mon[23750]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:00:07.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:07 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:07.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-mon[25838]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:00:07.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-mon[25838]: osdmap e53: 8 total, 7 up, 8 in 2024-09-17T01:00:07.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:07.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-mon[25838]: from='osd.7 [v2:172.21.15.100:6824/3730782588,v1:172.21.15.100:6825/3730782588]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:00:07.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-mon[25838]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:00:07.327 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:07.328 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[39033]: 2024-09-17T01:00:07.059+0000 7f86fd17e700 -1 osd.7 0 waiting for initial osdmap 2024-09-17T01:00:07.328 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:00:07 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[39033]: 2024-09-17T01:00:07.066+0000 7f86f7314700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:00:07.887 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: osd.7 [v2:172.21.15.100:6824/3730782588,v1:172.21.15.100:6825/3730782588] boot 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:08.313 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:08 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/802005502' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]': finished 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: osdmap e54: 8 total, 7 up, 8 in 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: osd.7 [v2:172.21.15.100:6824/3730782588,v1:172.21.15.100:6825/3730782588] boot 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: osdmap e55: 8 total, 8 up, 8 in 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:00:08.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:08 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/802005502' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-17T01:00:08.743 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1726534807,"num_in_osds":8,"osd_in_since":1726534790,"num_remapped_pgs":1} 2024-09-17T01:00:08.743 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T01:00:08.961 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:09.307 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:09 smithi033 ceph-mon[23750]: purged_snaps scrub starts 2024-09-17T01:00:09.307 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:09 smithi033 ceph-mon[23750]: purged_snaps scrub ok 2024-09-17T01:00:09.307 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:09 smithi033 ceph-mon[23750]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T01:00:09.471 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:09 smithi100 ceph-mon[25838]: purged_snaps scrub starts 2024-09-17T01:00:09.471 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:09 smithi100 ceph-mon[25838]: purged_snaps scrub ok 2024-09-17T01:00:09.471 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:09 smithi100 ceph-mon[25838]: osdmap e56: 8 total, 8 up, 8 in 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:00:10.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:00:10.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:00:10.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:10 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:10.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: osdmap e57: 8 total, 8 up, 8 in 2024-09-17T01:00:10.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:00:10.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:10 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:00:10.724 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:00:10.724 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":58,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","created":"2024-09-17T00:54:17.023112+0000","modified":"2024-09-17T01:00:10.439226+0000","last_up_change":"2024-09-17T01:00:07.428541+0000","last_in_change":"2024-09-17T00:59:50.370668+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T00:58:09.322907+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"52dcaf68-ea3d-48d0-9b79-e75911d2e73a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6803","nonce":2618793696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6805","nonce":2618793696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6809","nonce":2618793696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6807","nonce":2618793696}]},"public_addr":"172.21.15.33:6803/2618793696","cluster_addr":"172.21.15.33:6805/2618793696","heartbeat_back_addr":"172.21.15.33:6809/2618793696","heartbeat_front_addr":"172.21.15.33:6807/2618793696","state":["exists","up"]},{"osd":1,"uuid":"53834228-46f9-4bc6-9a87-e1f3ceecef40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6811","nonce":1083858096}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6813","nonce":1083858096}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6817","nonce":1083858096}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6815","nonce":1083858096}]},"public_addr":"172.21.15.33:6811/1083858096","cluster_addr":"172.21.15.33:6813/1083858096","heartbeat_back_addr":"172.21.15.33:6817/1083858096","heartbeat_front_addr":"172.21.15.33:6815/1083858096","state":["exists","up"]},{"osd":2,"uuid":"70c257f6-4971-449b-953b-4a301b9ed127","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6819","nonce":255012113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6821","nonce":255012113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6825","nonce":255012113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6823","nonce":255012113}]},"public_addr":"172.21.15.33:6819/255012113","cluster_addr":"172.21.15.33:6821/255012113","heartbeat_back_addr":"172.21.15.33:6825/255012113","heartbeat_front_addr":"172.21.15.33:6823/255012113","state":["exists","up"]},{"osd":3,"uuid":"cdd9da87-c04b-4213-a53c-39823c494cfa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6827","nonce":1776786389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6829","nonce":1776786389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6833","nonce":1776786389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6831","nonce":1776786389}]},"public_addr":"172.21.15.33:6827/1776786389","cluster_addr":"172.21.15.33:6829/1776786389","heartbeat_back_addr":"172.21.15.33:6833/1776786389","heartbeat_front_addr":"172.21.15.33:6831/1776786389","state":["exists","up"]},{"osd":4,"uuid":"a4848def-64f9-43f2-a0e3-0817e553032d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6800","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6801","nonce":588543990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6803","nonce":588543990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6807","nonce":588543990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6805","nonce":588543990}]},"public_addr":"172.21.15.100:6801/588543990","cluster_addr":"172.21.15.100:6803/588543990","heartbeat_back_addr":"172.21.15.100:6807/588543990","heartbeat_front_addr":"172.21.15.100:6805/588543990","state":["exists","up"]},{"osd":5,"uuid":"99c0933a-634c-4d6c-8bd4-dcd99e8121de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6809","nonce":2921136856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6811","nonce":2921136856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6815","nonce":2921136856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6813","nonce":2921136856}]},"public_addr":"172.21.15.100:6809/2921136856","cluster_addr":"172.21.15.100:6811/2921136856","heartbeat_back_addr":"172.21.15.100:6815/2921136856","heartbeat_front_addr":"172.21.15.100:6813/2921136856","state":["exists","up"]},{"osd":6,"uuid":"a78e41af-3b2b-4546-b469-15276fce2c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6817","nonce":1888615208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6819","nonce":1888615208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6823","nonce":1888615208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6821","nonce":1888615208}]},"public_addr":"172.21.15.100:6817/1888615208","cluster_addr":"172.21.15.100:6819/1888615208","heartbeat_back_addr":"172.21.15.100:6823/1888615208","heartbeat_front_addr":"172.21.15.100:6821/1888615208","state":["exists","up"]},{"osd":7,"uuid":"7b427a09-c6cf-4a16-9f36-44f98f7605a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6825","nonce":3730782588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6827","nonce":3730782588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6831","nonce":3730782588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6829","nonce":3730782588}]},"public_addr":"172.21.15.100:6825/3730782588","cluster_addr":"172.21.15.100:6827/3730782588","heartbeat_back_addr":"172.21.15.100:6831/3730782588","heartbeat_front_addr":"172.21.15.100:6829/3730782588","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:57:10.995092+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:57:40.595040+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:06.181153+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:31.503766+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:53.137904+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:59:16.675961+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:59:41.457748+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:00:06.109896+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:6800/2186199243":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/1676871229":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/576431843":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/3428827071":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/1887559003":"2024-09-18T00:55:12.332239+0000","172.21.15.33:6801/2186199243":"2024-09-18T00:56:00.585433+0000","172.21.15.33:6801/439692884":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/2680583486":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/2591781791":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/1858975160":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/3576249291":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/3894238799":"2024-09-18T00:56:00.585433+0000","172.21.15.33:6800/439692884":"2024-09-18T00:55:12.332239+0000","172.21.15.33:6800/3840895441":"2024-09-18T00:54:43.887602+0000","172.21.15.33:6801/3840895441":"2024-09-18T00:54:43.887602+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T01:00:11.267 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:11 smithi033 ceph-mon[23750]: Detected new or changed devices on smithi100 2024-09-17T01:00:11.268 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:11 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi100 to 3983M 2024-09-17T01:00:11.268 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:11 smithi033 ceph-mon[23750]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T01:00:11.268 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:11 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2896812417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:00:11.295 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-17T00:58:09.322907+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-17T01:00:11.295 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-17T01:00:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:11 smithi100 ceph-mon[25838]: Detected new or changed devices on smithi100 2024-09-17T01:00:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:11 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi100 to 3983M 2024-09-17T01:00:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:11 smithi100 ceph-mon[25838]: osdmap e58: 8 total, 8 up, 8 in 2024-09-17T01:00:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:11 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2896812417' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:00:11.515 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:12 smithi033 ceph-mon[23750]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:12.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:12 smithi033 ceph-mon[23750]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T01:00:12.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:12 smithi100 ceph-mon[25838]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:12.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:12 smithi100 ceph-mon[25838]: osdmap e59: 8 total, 8 up, 8 in 2024-09-17T01:00:13.109 INFO:teuthology.orchestra.run.smithi033.stdout:pg_num: 1 2024-09-17T01:00:13.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:13 smithi100 ceph-mon[25838]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:13.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:13 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/127751336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T01:00:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:13 smithi033 ceph-mon[23750]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:13.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:13 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/127751336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-17T01:00:13.959 INFO:tasks.cephadm:Setting up client nodes... 2024-09-17T01:00:13.959 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-17T01:00:15.932 INFO:teuthology.orchestra.run.smithi033.stdout:[client.0] 2024-09-17T01:00:15.932 INFO:teuthology.orchestra.run.smithi033.stdout: key = AQCf1OhmE9aVNxAA3llhMKR/PPOWCbucNaoHOg== 2024-09-17T01:00:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:15 smithi100 ceph-mon[25838]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:00:16.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:15 smithi033 ceph-mon[23750]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-17T01:00:16.645 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-17T01:00:16.646 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-17T01:00:16.646 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-17T01:00:16.686 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-17T01:00:16.686 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-17T01:00:16.686 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-17T01:00:16.944 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:17.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:16 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/199572177' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:00:17.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:16 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/199572177' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:00:17.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:16 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/199572177' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-17T01:00:17.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:16 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/199572177' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-17T01:00:18.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:17 smithi033 ceph-mon[23750]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-17T01:00:18.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:17 smithi100 ceph-mon[25838]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-17T01:00:18.723 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:00:19.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:18 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2293612189' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:00:19.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:18 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2293612189' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-17T01:00:19.617 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":19,"active_gid":14202,"active_name":"smithi033.illmog","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":901809280},{"type":"v1","addr":"172.21.15.33:6801","nonce":901809280}]},"active_addr":"172.21.15.33:6801/901809280","active_change":"2024-09-17T00:56:00.585724+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14222,"name":"smithi100.qrwkie","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.33:8443/","prometheus":"http://172.21.15.33:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3117290170}]},{"addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3520169578}]},{"addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":1887422551}]},{"addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":1525773166}]}]}} 2024-09-17T01:00:19.625 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-17T01:00:19.625 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-17T01:00:19.625 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T01:00:19.841 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:19 smithi100 ceph-mon[25838]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-17T01:00:20.253 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:19 smithi033 ceph-mon[23750]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-17T01:00:21.638 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:00:21.639 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":59,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","created":"2024-09-17T00:54:17.023112+0000","modified":"2024-09-17T01:00:11.441024+0000","last_up_change":"2024-09-17T01:00:07.428541+0000","last_in_change":"2024-09-17T00:59:50.370668+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T00:58:09.322907+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"52dcaf68-ea3d-48d0-9b79-e75911d2e73a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6803","nonce":2618793696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6805","nonce":2618793696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6809","nonce":2618793696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6807","nonce":2618793696}]},"public_addr":"172.21.15.33:6803/2618793696","cluster_addr":"172.21.15.33:6805/2618793696","heartbeat_back_addr":"172.21.15.33:6809/2618793696","heartbeat_front_addr":"172.21.15.33:6807/2618793696","state":["exists","up"]},{"osd":1,"uuid":"53834228-46f9-4bc6-9a87-e1f3ceecef40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6811","nonce":1083858096}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6813","nonce":1083858096}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6817","nonce":1083858096}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6815","nonce":1083858096}]},"public_addr":"172.21.15.33:6811/1083858096","cluster_addr":"172.21.15.33:6813/1083858096","heartbeat_back_addr":"172.21.15.33:6817/1083858096","heartbeat_front_addr":"172.21.15.33:6815/1083858096","state":["exists","up"]},{"osd":2,"uuid":"70c257f6-4971-449b-953b-4a301b9ed127","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6819","nonce":255012113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6821","nonce":255012113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6825","nonce":255012113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6823","nonce":255012113}]},"public_addr":"172.21.15.33:6819/255012113","cluster_addr":"172.21.15.33:6821/255012113","heartbeat_back_addr":"172.21.15.33:6825/255012113","heartbeat_front_addr":"172.21.15.33:6823/255012113","state":["exists","up"]},{"osd":3,"uuid":"cdd9da87-c04b-4213-a53c-39823c494cfa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6827","nonce":1776786389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6829","nonce":1776786389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6833","nonce":1776786389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6831","nonce":1776786389}]},"public_addr":"172.21.15.33:6827/1776786389","cluster_addr":"172.21.15.33:6829/1776786389","heartbeat_back_addr":"172.21.15.33:6833/1776786389","heartbeat_front_addr":"172.21.15.33:6831/1776786389","state":["exists","up"]},{"osd":4,"uuid":"a4848def-64f9-43f2-a0e3-0817e553032d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6800","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6801","nonce":588543990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6803","nonce":588543990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6807","nonce":588543990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6805","nonce":588543990}]},"public_addr":"172.21.15.100:6801/588543990","cluster_addr":"172.21.15.100:6803/588543990","heartbeat_back_addr":"172.21.15.100:6807/588543990","heartbeat_front_addr":"172.21.15.100:6805/588543990","state":["exists","up"]},{"osd":5,"uuid":"99c0933a-634c-4d6c-8bd4-dcd99e8121de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6809","nonce":2921136856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6811","nonce":2921136856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6815","nonce":2921136856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6813","nonce":2921136856}]},"public_addr":"172.21.15.100:6809/2921136856","cluster_addr":"172.21.15.100:6811/2921136856","heartbeat_back_addr":"172.21.15.100:6815/2921136856","heartbeat_front_addr":"172.21.15.100:6813/2921136856","state":["exists","up"]},{"osd":6,"uuid":"a78e41af-3b2b-4546-b469-15276fce2c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6817","nonce":1888615208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6819","nonce":1888615208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6823","nonce":1888615208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6821","nonce":1888615208}]},"public_addr":"172.21.15.100:6817/1888615208","cluster_addr":"172.21.15.100:6819/1888615208","heartbeat_back_addr":"172.21.15.100:6823/1888615208","heartbeat_front_addr":"172.21.15.100:6821/1888615208","state":["exists","up"]},{"osd":7,"uuid":"7b427a09-c6cf-4a16-9f36-44f98f7605a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6825","nonce":3730782588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6827","nonce":3730782588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6831","nonce":3730782588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6829","nonce":3730782588}]},"public_addr":"172.21.15.100:6825/3730782588","cluster_addr":"172.21.15.100:6827/3730782588","heartbeat_back_addr":"172.21.15.100:6831/3730782588","heartbeat_front_addr":"172.21.15.100:6829/3730782588","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:57:10.995092+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:57:40.595040+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:06.181153+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:31.503766+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:53.137904+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:59:16.675961+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:59:41.457748+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:00:06.109896+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:6800/2186199243":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/1676871229":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/576431843":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/3428827071":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/1887559003":"2024-09-18T00:55:12.332239+0000","172.21.15.33:6801/2186199243":"2024-09-18T00:56:00.585433+0000","172.21.15.33:6801/439692884":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/2680583486":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/2591781791":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/1858975160":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/3576249291":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/3894238799":"2024-09-18T00:56:00.585433+0000","172.21.15.33:6800/439692884":"2024-09-18T00:55:12.332239+0000","172.21.15.33:6800/3840895441":"2024-09-18T00:54:43.887602+0000","172.21.15.33:6801/3840895441":"2024-09-18T00:54:43.887602+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T01:00:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:21 smithi100 ceph-mon[25838]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:00:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:21 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/218239446' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:00:22.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:21 smithi033 ceph-mon[23750]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-17T01:00:22.322 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:21 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/218239446' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:00:22.346 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-17T01:00:22.347 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-17T01:00:22.562 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:23.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:23 smithi033 ceph-mon[23750]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-09-17T01:00:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:23 smithi100 ceph-mon[25838]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s, 0 objects/s recovering 2024-09-17T01:00:24.370 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:00:24.370 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":59,"fsid":"47eab686-748f-11ef-bceb-c7b262605968","created":"2024-09-17T00:54:17.023112+0000","modified":"2024-09-17T01:00:11.441024+0000","last_up_change":"2024-09-17T01:00:07.428541+0000","last_in_change":"2024-09-17T00:59:50.370668+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-17T00:58:09.322907+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"52dcaf68-ea3d-48d0-9b79-e75911d2e73a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6803","nonce":2618793696}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6805","nonce":2618793696}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6809","nonce":2618793696}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":2618793696},{"type":"v1","addr":"172.21.15.33:6807","nonce":2618793696}]},"public_addr":"172.21.15.33:6803/2618793696","cluster_addr":"172.21.15.33:6805/2618793696","heartbeat_back_addr":"172.21.15.33:6809/2618793696","heartbeat_front_addr":"172.21.15.33:6807/2618793696","state":["exists","up"]},{"osd":1,"uuid":"53834228-46f9-4bc6-9a87-e1f3ceecef40","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6811","nonce":1083858096}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6813","nonce":1083858096}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6817","nonce":1083858096}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":1083858096},{"type":"v1","addr":"172.21.15.33:6815","nonce":1083858096}]},"public_addr":"172.21.15.33:6811/1083858096","cluster_addr":"172.21.15.33:6813/1083858096","heartbeat_back_addr":"172.21.15.33:6817/1083858096","heartbeat_front_addr":"172.21.15.33:6815/1083858096","state":["exists","up"]},{"osd":2,"uuid":"70c257f6-4971-449b-953b-4a301b9ed127","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6819","nonce":255012113}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6821","nonce":255012113}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6825","nonce":255012113}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":255012113},{"type":"v1","addr":"172.21.15.33:6823","nonce":255012113}]},"public_addr":"172.21.15.33:6819/255012113","cluster_addr":"172.21.15.33:6821/255012113","heartbeat_back_addr":"172.21.15.33:6825/255012113","heartbeat_front_addr":"172.21.15.33:6823/255012113","state":["exists","up"]},{"osd":3,"uuid":"cdd9da87-c04b-4213-a53c-39823c494cfa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6827","nonce":1776786389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6829","nonce":1776786389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6833","nonce":1776786389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":1776786389},{"type":"v1","addr":"172.21.15.33:6831","nonce":1776786389}]},"public_addr":"172.21.15.33:6827/1776786389","cluster_addr":"172.21.15.33:6829/1776786389","heartbeat_back_addr":"172.21.15.33:6833/1776786389","heartbeat_front_addr":"172.21.15.33:6831/1776786389","state":["exists","up"]},{"osd":4,"uuid":"a4848def-64f9-43f2-a0e3-0817e553032d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6800","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6801","nonce":588543990}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6802","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6803","nonce":588543990}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6806","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6807","nonce":588543990}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6804","nonce":588543990},{"type":"v1","addr":"172.21.15.100:6805","nonce":588543990}]},"public_addr":"172.21.15.100:6801/588543990","cluster_addr":"172.21.15.100:6803/588543990","heartbeat_back_addr":"172.21.15.100:6807/588543990","heartbeat_front_addr":"172.21.15.100:6805/588543990","state":["exists","up"]},{"osd":5,"uuid":"99c0933a-634c-4d6c-8bd4-dcd99e8121de","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6808","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6809","nonce":2921136856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6810","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6811","nonce":2921136856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6814","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6815","nonce":2921136856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6812","nonce":2921136856},{"type":"v1","addr":"172.21.15.100:6813","nonce":2921136856}]},"public_addr":"172.21.15.100:6809/2921136856","cluster_addr":"172.21.15.100:6811/2921136856","heartbeat_back_addr":"172.21.15.100:6815/2921136856","heartbeat_front_addr":"172.21.15.100:6813/2921136856","state":["exists","up"]},{"osd":6,"uuid":"a78e41af-3b2b-4546-b469-15276fce2c91","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6816","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6817","nonce":1888615208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6818","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6819","nonce":1888615208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6822","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6823","nonce":1888615208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6820","nonce":1888615208},{"type":"v1","addr":"172.21.15.100:6821","nonce":1888615208}]},"public_addr":"172.21.15.100:6817/1888615208","cluster_addr":"172.21.15.100:6819/1888615208","heartbeat_back_addr":"172.21.15.100:6823/1888615208","heartbeat_front_addr":"172.21.15.100:6821/1888615208","state":["exists","up"]},{"osd":7,"uuid":"7b427a09-c6cf-4a16-9f36-44f98f7605a7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6824","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6825","nonce":3730782588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6826","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6827","nonce":3730782588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6830","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6831","nonce":3730782588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.100:6828","nonce":3730782588},{"type":"v1","addr":"172.21.15.100:6829","nonce":3730782588}]},"public_addr":"172.21.15.100:6825/3730782588","cluster_addr":"172.21.15.100:6827/3730782588","heartbeat_back_addr":"172.21.15.100:6831/3730782588","heartbeat_front_addr":"172.21.15.100:6829/3730782588","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:57:10.995092+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:57:40.595040+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:06.181153+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:31.503766+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:58:53.137904+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:59:16.675961+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T00:59:41.457748+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-17T01:00:06.109896+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:6800/2186199243":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/1676871229":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/576431843":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/3428827071":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/1887559003":"2024-09-18T00:55:12.332239+0000","172.21.15.33:6801/2186199243":"2024-09-18T00:56:00.585433+0000","172.21.15.33:6801/439692884":"2024-09-18T00:55:12.332239+0000","172.21.15.33:0/2680583486":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/2591781791":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/1858975160":"2024-09-18T00:56:00.585433+0000","172.21.15.33:0/3576249291":"2024-09-18T00:54:43.887602+0000","172.21.15.33:0/3894238799":"2024-09-18T00:56:00.585433+0000","172.21.15.33:6800/439692884":"2024-09-18T00:55:12.332239+0000","172.21.15.33:6800/3840895441":"2024-09-18T00:54:43.887602+0000","172.21.15.33:6801/3840895441":"2024-09-18T00:54:43.887602+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-17T01:00:24.685 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:24 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3713986802' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:00:24.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:24 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3713986802' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-17T01:00:24.995 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-17T01:00:24.995 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-17T01:00:24.996 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-17T01:00:24.996 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-17T01:00:24.996 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-17T01:00:24.997 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-17T01:00:24.997 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-17T01:00:24.998 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-17T01:00:25.425 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.425 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.426 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.427 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.432 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.438 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.462 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.463 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:25.700 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:25 smithi033 ceph-mon[23750]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T01:00:25.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:25 smithi100 ceph-mon[25838]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2024-09-17T01:00:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:27 smithi033 ceph-mon[23750]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:28.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:27 smithi100 ceph-mon[25838]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:30.169 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:29 smithi033 ceph-mon[23750]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:29 smithi100 ceph-mon[25838]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:32.071 INFO:teuthology.orchestra.run.smithi033.stdout:111669149723 2024-09-17T01:00:32.071 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-17T01:00:32.173 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:31 smithi033 ceph-mon[23750]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:31 smithi100 ceph-mon[25838]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:33.559 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:33 smithi033 ceph-mon[23750]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:33 smithi100 ceph-mon[25838]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:35.521 INFO:teuthology.orchestra.run.smithi033.stdout:201863462924 2024-09-17T01:00:35.521 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-17T01:00:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:35 smithi100 ceph-mon[25838]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:36.267 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:35 smithi033 ceph-mon[23750]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:37.727 INFO:teuthology.orchestra.run.smithi033.stdout:38654705708 2024-09-17T01:00:37.728 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-17T01:00:38.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:37 smithi033 ceph-mon[23750]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:37 smithi100 ceph-mon[25838]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:39.028 INFO:teuthology.orchestra.run.smithi033.stdout:133143986199 2024-09-17T01:00:39.028 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-17T01:00:39.500 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:39.695 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:39.696 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:40.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:39 smithi033 ceph-mon[23750]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:40.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:39 smithi100 ceph-mon[25838]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:41.210 INFO:teuthology.orchestra.run.smithi033.stdout:60129542183 2024-09-17T01:00:41.211 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-17T01:00:42.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:41 smithi033 ceph-mon[23750]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:41 smithi100 ceph-mon[25838]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:43.254 INFO:teuthology.orchestra.run.smithi033.stdout:81604378658 2024-09-17T01:00:43.255 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-17T01:00:43.522 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:43 smithi033 ceph-mon[23750]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:43.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:43 smithi100 ceph-mon[25838]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:44.087 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:44.486 INFO:teuthology.orchestra.run.smithi033.stdout:236223201289 2024-09-17T01:00:44.486 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-17T01:00:44.519 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:44.519 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:45.334 INFO:teuthology.orchestra.run.smithi033.stdout:201863462927 2024-09-17T01:00:45.662 INFO:teuthology.orchestra.run.smithi033.stdout:167503724561 2024-09-17T01:00:45.662 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-17T01:00:46.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:45 smithi033 ceph-mon[23750]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:46.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:45 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1979689652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T01:00:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:45 smithi100 ceph-mon[25838]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:45 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1979689652' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-17T01:00:48.095 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:47 smithi033 ceph-mon[23750]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:48.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:47 smithi100 ceph-mon[25838]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:48.630 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462924 got 201863462927 for osd.6 2024-09-17T01:00:48.631 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:49.057 INFO:teuthology.orchestra.run.smithi033.stdout:133143986201 2024-09-17T01:00:50.061 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:49 smithi033 ceph-mon[23750]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:50.062 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:49 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2797822928' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T01:00:50.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:49 smithi100 ceph-mon[25838]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:50.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:49 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2797822928' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-17T01:00:50.490 INFO:teuthology.orchestra.run.smithi033.stdout:81604378659 2024-09-17T01:00:51.071 INFO:teuthology.orchestra.run.smithi033.stdout:38654705711 2024-09-17T01:00:51.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:50 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/425350199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T01:00:51.261 INFO:teuthology.orchestra.run.smithi033.stdout:111669149727 2024-09-17T01:00:51.261 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:50 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/425350199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-17T01:00:51.956 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:52.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:51 smithi033 ceph-mon[23750]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:52.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:51 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2565674087' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T01:00:52.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:51 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1910854134' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T01:00:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:51 smithi100 ceph-mon[25838]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:51 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2565674087' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-17T01:00:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:51 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1910854134' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-17T01:00:53.001 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378658 got 81604378659 for osd.2 2024-09-17T01:00:53.001 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:53.229 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:00:53.532 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:53 smithi033 ceph-mon[23750]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:53.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:53 smithi100 ceph-mon[25838]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:54.244 INFO:teuthology.orchestra.run.smithi033.stdout:60129542185 2024-09-17T01:00:54.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:54 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1815641296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T01:00:54.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:54 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1815641296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-17T01:00:55.018 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986199 got 133143986201 for osd.4 2024-09-17T01:00:55.018 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:55.187 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705708 got 38654705711 for osd.0 2024-09-17T01:00:55.188 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:55.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:55 smithi033 ceph-mon[23750]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:55.554 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149723 got 111669149727 for osd.3 2024-09-17T01:00:55.554 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:55.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:55 smithi100 ceph-mon[25838]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:56.353 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542185 for osd.1 2024-09-17T01:00:56.353 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:58.000 INFO:teuthology.orchestra.run.smithi033.stdout:236223201292 2024-09-17T01:00:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:57 smithi033 ceph-mon[23750]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:58.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:57 smithi100 ceph-mon[25838]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:00:58.450 INFO:teuthology.orchestra.run.smithi033.stdout:167503724565 2024-09-17T01:00:59.075 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:58 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1638740679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T01:00:59.076 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:58 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1071544465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T01:00:59.103 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201289 got 236223201292 for osd.7 2024-09-17T01:00:59.103 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:59.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:58 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1638740679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-17T01:00:59.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:58 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1071544465' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-17T01:00:59.483 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724561 got 167503724565 for osd.5 2024-09-17T01:00:59.483 DEBUG:teuthology.parallel:result is None 2024-09-17T01:00:59.484 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-17T01:00:59.484 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T01:00:59.714 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:01:00.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:00:59 smithi033 ceph-mon[23750]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:00.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:00:59 smithi100 ceph-mon[25838]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:01:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:01:01.312 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:01.313 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:01:01.313 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:01:01.314 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-17T01:01:01.911 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":197,"stamp":"2024-09-17T01:01:00.876826+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48796,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912420,"statfs":{"total":767960285184,"available":767910318080,"internally_reserved":0,"allocated":6451200,"data_stored":3546561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002042"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-17T01:00:11.449278+0000","last_change":"2024-09-17T01:00:11.449278+0000","last_active":"2024-09-17T01:00:11.449278+0000","last_peered":"2024-09-17T01:00:11.449278+0000","last_clean":"2024-09-17T01:00:11.449278+0000","last_became_active":"2024-09-17T01:00:11.448853+0000","last_became_peered":"2024-09-17T01:00:11.448853+0000","last_unstale":"2024-09-17T01:00:11.449278+0000","last_undegraded":"2024-09-17T01:00:11.449278+0000","last_fullsized":"2024-09-17T01:00:11.449278+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T00:58:09.323293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T00:58:09.323293+0000","last_clean_scrub_stamp":"2024-09-17T00:58:09.323293+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T11:41:43.758926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58599999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69099999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60099999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.53800000000000003}]},{"osd":1,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.505},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51900000000000002}]},{"osd":2,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.61399999999999999}]},{"osd":3,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.64600000000000002}]},{"osd":4,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.63400000000000001}]},{"osd":5,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.68400000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69299999999999995}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38500000000000001,"15min":0.36399999999999999},"min":{"1min":0.247,"5min":0.221,"15min":0.221},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.317},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.35699999999999998},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.34799999999999998}]},{"osd":2,"last update":"Tue Sep 17 01:00:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41299999999999998,"15min":0.40500000000000003},"min":{"1min":0.31900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.378}]},{"osd":3,"last update":"Tue Sep 17 01:00:41 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.48799999999999999,"15min":0.48499999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.40999999999999998}]},{"osd":4,"last update":"Tue Sep 17 00:59:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.56899999999999995}]},{"osd":5,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.505}]},{"osd":6,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.64700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 01:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.38,"15min":0.36599999999999999},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.63600000000000001,"5min":0.754,"15min":0.754},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.373,"15min":0.35899999999999999},"min":{"1min":0.26600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56200000000000006,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.30299999999999999}]},{"osd":2,"last update":"Tue Sep 17 01:00:15 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.434,"15min":0.433},"min":{"1min":0.308,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58299999999999996,"5min":0.875,"15min":0.875},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.435,"15min":0.435},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.55600000000000005,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.34399999999999997}]},{"osd":3,"last update":"Tue Sep 17 01:00:38 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46100000000000002,"15min":0.46000000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.433},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.46899999999999997,"15min":0.46600000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.31900000000000001}]},{"osd":4,"last update":"Tue Sep 17 00:59:58 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55100000000000005}]},{"osd":5,"last update":"Tue Sep 17 01:00:20 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.51500000000000001}]},{"osd":6,"last update":"Tue Sep 17 01:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.56899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.36299999999999999}]},{"osd":1,"last update":"Tue Sep 17 01:00:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.314},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41799999999999998,"15min":0.40999999999999998},"min":{"1min":0.32900000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.35599999999999998}]},{"osd":3,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.48099999999999998,"15min":0.48899999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.30199999999999999}]},{"osd":4,"last update":"Tue Sep 17 01:00:57 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.093,"15min":1.171},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.034,"5min":15.801,"15min":15.801},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.64100000000000001,"15min":0.63800000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.68999999999999995}]},{"osd":5,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.67000000000000004}]},{"osd":6,"last update":"Tue Sep 17 01:00:43 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.65100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59899999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39500000000000002,"15min":0.39400000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.317},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.433,"15min":0.438},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.68999999999999995,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.36699999999999999}]},{"osd":1,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44800000000000001,"15min":0.45600000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54600000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.42599999999999999,"15min":0.42999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.52600000000000002,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.375}]},{"osd":2,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65200000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.435},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.45900000000000002,"15min":0.46400000000000002},"min":{"1min":0.32200000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.60099999999999998,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.35099999999999998}]},{"osd":4,"last update":"Tue Sep 17 00:59:59 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53000000000000003}]},{"osd":5,"last update":"Tue Sep 17 01:00:19 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.57099999999999995}]},{"osd":6,"last update":"Tue Sep 17 01:00:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.64600000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986203,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.53700000000000003,"15min":0.53300000000000003},"min":{"1min":0.42299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.371,"15min":0.371},"max":{"1min":0.66300000000000003,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.58999999999999997}]},{"osd":1,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73699999999999999,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53400000000000003,"15min":0.53200000000000003},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.68700000000000006,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.60499999999999998}]},{"osd":2,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.443,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71099999999999997,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53600000000000003,"15min":0.53500000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69599999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.54300000000000004}]},{"osd":3,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.47699999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.55700000000000005,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77500000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.61599999999999999}]},{"osd":5,"last update":"Tue Sep 17 01:00:19 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.56599999999999995}]},{"osd":6,"last update":"Tue Sep 17 01:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.47599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52900000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724566,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.55300000000000005}]},{"osd":1,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.627},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.65600000000000003}]},{"osd":2,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.749}]},{"osd":3,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.56599999999999995}]},{"osd":4,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.58299999999999996}]},{"osd":6,"last update":"Tue Sep 17 01:00:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T01:01:01.913 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-17T01:01:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:01 smithi033 ceph-mon[23750]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:02.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:01 smithi033 ceph-mon[23750]: from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:02.134 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:01:02.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:01 smithi100 ceph-mon[25838]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:02.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:01 smithi100 ceph-mon[25838]: from='client.14438 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:03.532 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:03 smithi033 ceph-mon[23750]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:03 smithi100 ceph-mon[25838]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:03.817 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:03.819 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-17T01:01:04.392 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":198,"stamp":"2024-09-17T01:01:02.877228+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48796,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42432,"kb_avail":749912420,"statfs":{"total":767960285184,"available":767910318080,"internally_reserved":0,"allocated":6451200,"data_stored":3546561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43450368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002050"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-17T01:00:11.449278+0000","last_change":"2024-09-17T01:00:11.449278+0000","last_active":"2024-09-17T01:00:11.449278+0000","last_peered":"2024-09-17T01:00:11.449278+0000","last_clean":"2024-09-17T01:00:11.449278+0000","last_became_active":"2024-09-17T01:00:11.448853+0000","last_became_peered":"2024-09-17T01:00:11.448853+0000","last_unstale":"2024-09-17T01:00:11.449278+0000","last_undegraded":"2024-09-17T01:00:11.449278+0000","last_fullsized":"2024-09-17T01:00:11.449278+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-17T00:58:09.323293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-17T00:58:09.323293+0000","last_clean_scrub_stamp":"2024-09-17T00:58:09.323293+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T11:41:43.758926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3194880,"data_stored":3173568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201293,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58599999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65600000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69099999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60099999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53500000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.53800000000000003}]},{"osd":1,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.505},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51900000000000002}]},{"osd":2,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.61399999999999999}]},{"osd":3,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.64600000000000002}]},{"osd":4,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.63400000000000001}]},{"osd":5,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.68400000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69299999999999995}]}]},{"osd":1,"up_from":14,"seq":60129542187,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:45 2024","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.38500000000000001,"15min":0.36399999999999999},"min":{"1min":0.247,"5min":0.221,"15min":0.221},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.317},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.38400000000000001,"15min":0.35699999999999998},"min":{"1min":0.26600000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.34799999999999998}]},{"osd":2,"last update":"Tue Sep 17 01:00:13 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.41299999999999998,"15min":0.40500000000000003},"min":{"1min":0.31900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.29299999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.378}]},{"osd":3,"last update":"Tue Sep 17 01:00:41 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.48799999999999999,"15min":0.48499999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.40999999999999998}]},{"osd":4,"last update":"Tue Sep 17 00:59:56 2024","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.56899999999999995}]},{"osd":5,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.505}]},{"osd":6,"last update":"Tue Sep 17 01:00:42 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.077,"5min":1.077,"15min":1.077},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.64700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61699999999999999}]}]},{"osd":0,"up_from":9,"seq":38654705713,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6884,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738268,"statfs":{"total":95995035648,"available":95987986432,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Sep 17 01:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.38,"15min":0.36599999999999999},"min":{"1min":0.29299999999999998,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.63600000000000001,"5min":0.754,"15min":0.754},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.373,"15min":0.35899999999999999},"min":{"1min":0.26600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.56200000000000006,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.30299999999999999}]},{"osd":2,"last update":"Tue Sep 17 01:00:15 2024","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.434,"15min":0.433},"min":{"1min":0.308,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58299999999999996,"5min":0.875,"15min":0.875},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.438,"5min":0.435,"15min":0.435},"min":{"1min":0.33400000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.55600000000000005,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.34399999999999997}]},{"osd":3,"last update":"Tue Sep 17 01:00:38 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46100000000000002,"15min":0.46000000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.433},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.46899999999999997,"15min":0.46600000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.31900000000000001}]},{"osd":4,"last update":"Tue Sep 17 00:59:58 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.55100000000000005}]},{"osd":5,"last update":"Tue Sep 17 01:00:20 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.96599999999999997,"5min":0.96599999999999997,"15min":0.96599999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.51500000000000001}]},{"osd":6,"last update":"Tue Sep 17 01:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.56899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6236,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738916,"statfs":{"total":95995035648,"available":95988649984,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.29799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.36299999999999999}]},{"osd":1,"last update":"Tue Sep 17 01:00:12 2024","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.38600000000000001,"15min":0.38400000000000001},"min":{"1min":0.28899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.314},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41799999999999998,"15min":0.40999999999999998},"min":{"1min":0.32900000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"last":0.35599999999999998}]},{"osd":3,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.437,"5min":0.48099999999999998,"15min":0.48899999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.30199999999999999}]},{"osd":4,"last update":"Tue Sep 17 01:00:57 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.093,"15min":1.171},"min":{"1min":0.36099999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":1.034,"5min":15.801,"15min":15.801},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.64100000000000001,"15min":0.63800000000000001},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.044,"5min":1.044,"15min":1.044},"last":0.68999999999999995}]},{"osd":5,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.67000000000000004}]},{"osd":6,"last update":"Tue Sep 17 01:00:43 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.97199999999999998,"5min":0.97199999999999998,"15min":0.97199999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.65100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59899999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149729,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39500000000000002,"15min":0.39400000000000002},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.317},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.433,"15min":0.438},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.68999999999999995,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.36699999999999999}]},{"osd":1,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.44800000000000001,"15min":0.45600000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54600000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.42599999999999999,"15min":0.42999999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.52600000000000002,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.375}]},{"osd":2,"last update":"Tue Sep 17 01:00:37 2024","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65200000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.435},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.45900000000000002,"15min":0.46400000000000002},"min":{"1min":0.32200000000000001,"5min":0.313,"15min":0.313},"max":{"1min":0.60099999999999998,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.35099999999999998}]},{"osd":4,"last update":"Tue Sep 17 00:59:59 2024","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53000000000000003}]},{"osd":5,"last update":"Tue Sep 17 01:00:19 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.57099999999999995}]},{"osd":6,"last update":"Tue Sep 17 01:00:45 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.64600000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61199999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986204,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.53700000000000003,"15min":0.53300000000000003},"min":{"1min":0.42299999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.38700000000000001,"5min":0.371,"15min":0.371},"max":{"1min":0.66300000000000003,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.61299999999999999}]},{"osd":1,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.53500000000000003,"15min":0.52900000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73699999999999999,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.498},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53400000000000003,"15min":0.53200000000000003},"min":{"1min":0.39400000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.68700000000000006,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42899999999999999}]},{"osd":2,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.52500000000000002},"min":{"1min":0.443,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71099999999999997,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53600000000000003,"15min":0.53500000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69599999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.52600000000000002}]},{"osd":3,"last update":"Tue Sep 17 01:00:55 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.53500000000000003,"15min":0.52700000000000002},"min":{"1min":0.47699999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.55700000000000005,"15min":0.55400000000000005},"min":{"1min":0.40200000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.77500000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.63}]},{"osd":5,"last update":"Tue Sep 17 01:00:19 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.56699999999999995}]},{"osd":6,"last update":"Tue Sep 17 01:00:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.60099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724566,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5660,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739492,"statfs":{"total":95995035648,"available":95989239808,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.55300000000000005}]},{"osd":1,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.627},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.65600000000000003}]},{"osd":2,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.749}]},{"osd":3,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.56599999999999995}]},{"osd":4,"last update":"Tue Sep 17 01:00:18 2024","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.58299999999999996}]},{"osd":6,"last update":"Tue Sep 17 01:00:47 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-17T01:01:04.396 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-17T01:01:04.397 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-17T01:01:04.397 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-17T01:01:04.397 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-17T01:01:04.617 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/mon.smithi033/config 2024-09-17T01:01:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:04 smithi100 ceph-mon[25838]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:04.738 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:04 smithi033 ceph-mon[23750]: from='client.14442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:05.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:05 smithi100 ceph-mon[25838]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:05.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:05 smithi033 ceph-mon[23750]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:06.312 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:06.313 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-17T01:01:06.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:06 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/749776974' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T01:01:06.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:06 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/749776974' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-17T01:01:07.088 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-17T01:01:07.088 INFO:tasks.cephadm:Setup complete, yielding 2024-09-17T01:01:07.088 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:01:07.098 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:01:07.098 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-17T01:01:07.657 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:07 smithi033 ceph-mon[23750]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:07.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:07 smithi100 ceph-mon[25838]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:08.969 INFO:teuthology.orchestra.run.smithi033.stdout:Backend: cephadm 2024-09-17T01:01:08.969 INFO:teuthology.orchestra.run.smithi033.stdout:Available: Yes 2024-09-17T01:01:08.970 INFO:teuthology.orchestra.run.smithi033.stdout:Paused: No 2024-09-17T01:01:09.548 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-17T01:01:10.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:09 smithi033 ceph-mon[23750]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:10.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:09 smithi033 ceph-mon[23750]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:10.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:09 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:10.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:09 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:10.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:09 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:10.214 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:09 smithi100 ceph-mon[25838]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:10.214 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:09 smithi100 ceph-mon[25838]: from='client.14450 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:10.214 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:09 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:10.214 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:09 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:10.214 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:09 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:11.531 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:01:11.531 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (4m) 2m ago 5m 15.4M - ba2b418f427c a6247209f546 2024-09-17T01:01:11.531 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (5m) 64s ago 5m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (4m) 2m ago 5m 43.5M - 8.3.5 dad864ee21e9 e7569df2559d 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:9283 running (6m) 2m ago 6m 447M - 17.2.0 e1d6a67b021e 4df4282576a9 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (4m) 64s ago 4m 413M - 17.2.0 e1d6a67b021e 4c4fbb94047d 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (6m) 2m ago 6m 41.0M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (4m) 64s ago 4m 41.8M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (5m) 2m ago 5m 9.83M - 1dbe0e931976 65382f92c0e1 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (4m) 64s ago 4m 19.3M - 1dbe0e931976 7fb37007ed80 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (4m) 2m ago 4m 32.9M 3217M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (3m) 2m ago 3m 35.0M 3217M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (3m) 2m ago 3m 33.2M 3217M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (2m) 2m ago 2m 11.0M 3217M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:01:11.532 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (2m) 64s ago 2m 33.6M 3983M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:01:11.533 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (119s) 64s ago 118s 35.9M 3983M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:01:11.533 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (94s) 64s ago 94s 30.3M 3983M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:01:11.533 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (69s) 64s ago 69s 10.7M 3983M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:01:11.533 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (4m) 2m ago 4m 55.2M - 514e6a882f6e 3afe8f20ef9f 2024-09-17T01:01:11.543 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:11 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:11 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:12.262 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-17T01:01:12.573 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:12 smithi033 ceph-mon[23750]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:12 smithi100 ceph-mon[25838]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:13.552 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:13 smithi033 ceph-mon[23750]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:13.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:13 smithi100 ceph-mon[25838]: from='client.14454 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:14.228 INFO:teuthology.orchestra.run.smithi033.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-17T01:01:14.228 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-17T01:01:14.228 INFO:teuthology.orchestra.run.smithi033.stdout:crash 2/2 2m ago 6m * 2024-09-17T01:01:14.228 INFO:teuthology.orchestra.run.smithi033.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-17T01:01:14.229 INFO:teuthology.orchestra.run.smithi033.stdout:mgr 2/2 2m ago 6m count:2 2024-09-17T01:01:14.229 INFO:teuthology.orchestra.run.smithi033.stdout:mon 2/2 2m ago 5m smithi033:172.21.15.33=smithi033;smithi100:172.21.15.100=smithi100;count:2 2024-09-17T01:01:14.229 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-17T01:01:14.229 INFO:teuthology.orchestra.run.smithi033.stdout:osd 8 2m ago - 2024-09-17T01:01:14.229 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-17T01:01:14.534 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:14 smithi033 ceph-mon[23750]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:14 smithi100 ceph-mon[25838]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:14.896 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-17T01:01:15.496 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:15 smithi033 ceph-mon[23750]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:15.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:15 smithi100 ceph-mon[25838]: from='client.14458 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:16.358 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:16 smithi033 ceph-mon[23750]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:16 smithi100 ceph-mon[25838]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:16.753 INFO:teuthology.orchestra.run.smithi033.stdout:HOST ADDR LABELS STATUS 2024-09-17T01:01:16.753 INFO:teuthology.orchestra.run.smithi033.stdout:smithi033 172.21.15.33 2024-09-17T01:01:16.753 INFO:teuthology.orchestra.run.smithi033.stdout:smithi100 172.21.15.100 2024-09-17T01:01:16.753 INFO:teuthology.orchestra.run.smithi033.stdout:2 hosts in cluster 2024-09-17T01:01:17.374 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-17T01:01:18.588 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:18 smithi033 ceph-mon[23750]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:18.589 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:18 smithi033 ceph-mon[23750]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:18.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:18 smithi100 ceph-mon[25838]: from='client.14462 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:18.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:18 smithi100 ceph-mon[25838]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:19.304 INFO:teuthology.orchestra.run.smithi033.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-17T01:01:19.304 INFO:teuthology.orchestra.run.smithi033.stdout:smithi033 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT5330008Q400BGN 400G LVM detected, locked 2024-09-17T01:01:19.304 INFO:teuthology.orchestra.run.smithi033.stdout:smithi100 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_PHFT620400WR400BGN 400G LVM detected, locked 2024-09-17T01:01:19.896 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T01:01:19.907 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:01:19.908 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T01:01:19.936 INFO:teuthology.orchestra.run.smithi033.stderr:+ systemctl stop nfs-server 2024-09-17T01:01:19.944 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi100.front.sepia.ceph.com 2024-09-17T01:01:19.944 DEBUG:teuthology.orchestra.run.smithi100:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-17T01:01:19.974 INFO:teuthology.orchestra.run.smithi100.stderr:+ systemctl stop nfs-server 2024-09-17T01:01:19.983 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:01:19.991 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:01:19.991 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-17T01:01:20.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:20 smithi033 ceph-mon[23750]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:20.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:20 smithi033 ceph-mon[23750]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:20 smithi100 ceph-mon[25838]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:20 smithi100 ceph-mon[25838]: from='client.14466 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:22 smithi033 ceph-mon[23750]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T01:01:22.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:22 smithi100 ceph-mon[25838]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:22.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-17T01:01:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:23 smithi033 ceph-mon[23750]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T01:01:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:23 smithi033 ceph-mon[23750]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T01:01:23.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:23 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T01:01:23.611 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:23 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[23744]: 2024-09-17T01:01:23.320+0000 7f7f35665700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:01:23.629 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:23 smithi100 ceph-mon[25838]: from='client.14470 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:23.629 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-17T01:01:23.629 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:23 smithi100 ceph-mon[25838]: osdmap e60: 8 total, 8 up, 8 in 2024-09-17T01:01:23.629 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:23 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-17T01:01:23.976 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-17T01:01:23.985 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-17T01:01:23.986 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T01:01:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: pgmap v209: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T01:01:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T01:01:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T01:01:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: fsmap foofs:0 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: Saving service mds.foofs spec with placement count:2 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T01:01:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:24 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:24.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: pgmap v209: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:24.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-17T01:01:24.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: osdmap e61: 8 total, 8 up, 8 in 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: osdmap e62: 8 total, 8 up, 8 in 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: fsmap foofs:0 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: Saving service mds.foofs spec with placement count:2 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:24.498 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:01:24.499 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T01:01:24.499 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:24 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:25.571 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:25 smithi033 ceph-mon[23750]: Deploying daemon mds.foofs.smithi033.gofagb on smithi033 2024-09-17T01:01:25.571 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:25 smithi033 ceph-mon[23750]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T01:01:25.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:25 smithi100 ceph-mon[25838]: Deploying daemon mds.foofs.smithi033.gofagb on smithi033 2024-09-17T01:01:25.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:25 smithi100 ceph-mon[25838]: osdmap e63: 8 total, 8 up, 8 in 2024-09-17T01:01:26.562 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:26 smithi033 ceph-mon[23750]: pgmap v213: 65 pgs: 2 creating+activating, 28 creating+peering, 21 unknown, 14 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:26.562 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:26 smithi033 ceph-mon[23750]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T01:01:26.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:26 smithi100 ceph-mon[25838]: pgmap v213: 65 pgs: 2 creating+activating, 28 creating+peering, 21 unknown, 14 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:26.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:26 smithi100 ceph-mon[25838]: osdmap e64: 8 total, 8 up, 8 in 2024-09-17T01:01:26.885 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:26.885 INFO:teuthology.orchestra.run.smithi033.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T00:55:01.576222Z", "last_refresh": "2024-09-17T00:58:34.030725Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T00:54:55.903476Z", "last_refresh": "2024-09-17T00:58:34.030794Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T00:54:58.805315Z", "last_refresh": "2024-09-17T00:58:34.030858Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T01:01:23.342599Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T01:01:23.335227Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T00:54:54.644049Z", "last_refresh": "2024-09-17T00:58:34.030651Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi033:172.21.15.33=smithi033", "smithi100:172.21.15.100=smithi100"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T00:55:43.948307Z", "last_refresh": "2024-09-17T00:58:34.030509Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T00:55:00.041575Z", "last_refresh": "2024-09-17T00:58:34.030922Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T00:58:34.031069Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T00:54:57.172843Z", "last_refresh": "2024-09-17T00:58:34.031004Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T01:01:27.636 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T01:01:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: pgmap v215: 65 pgs: 2 creating+activating, 28 creating+peering, 16 unknown, 19 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: daemon mds.foofs.smithi033.gofagb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T01:01:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: Cluster is now healthy 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: mds.? [v2:172.21.15.33:6834/2928191622,v1:172.21.15.33:6835/2928191622] up:boot 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:creating} 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: daemon mds.foofs.smithi033.gofagb is now active in filesystem foofs as rank 0 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T01:01:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:28 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:28.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: pgmap v215: 65 pgs: 2 creating+activating, 28 creating+peering, 16 unknown, 19 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-17T01:01:28.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: from='client.14474 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:28.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: daemon mds.foofs.smithi033.gofagb assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: Cluster is now healthy 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: mds.? [v2:172.21.15.33:6834/2928191622,v1:172.21.15.33:6835/2928191622] up:boot 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:creating} 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: daemon mds.foofs.smithi033.gofagb is now active in filesystem foofs as rank 0 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-17T01:01:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:28 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:28.637 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T01:01:29.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:29 smithi033 ceph-mon[23750]: Deploying daemon mds.foofs.smithi100.ssfebu on smithi100 2024-09-17T01:01:29.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:29 smithi033 ceph-mon[23750]: mds.? [v2:172.21.15.33:6834/2928191622,v1:172.21.15.33:6835/2928191622] up:active 2024-09-17T01:01:29.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:29 smithi033 ceph-mon[23750]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 2024-09-17T01:01:29.662 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:29 smithi100 ceph-mon[25838]: Deploying daemon mds.foofs.smithi100.ssfebu on smithi100 2024-09-17T01:01:29.662 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:29 smithi100 ceph-mon[25838]: mds.? [v2:172.21.15.33:6834/2928191622,v1:172.21.15.33:6835/2928191622] up:active 2024-09-17T01:01:29.662 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:29 smithi100 ceph-mon[25838]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 2024-09-17T01:01:30.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:30 smithi033 ceph-mon[23750]: pgmap v216: 65 pgs: 2 creating+activating, 6 creating+peering, 57 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 920 B/s wr, 3 op/s 2024-09-17T01:01:30.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:30 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:30.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:30 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:30.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:30 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:30.439 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:30 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:30.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:30 smithi100 ceph-mon[25838]: pgmap v216: 65 pgs: 2 creating+activating, 6 creating+peering, 57 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 920 B/s wr, 3 op/s 2024-09-17T01:01:30.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:30 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:30.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:30 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:30.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:30 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:30.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:30 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:30.751 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:30.752 INFO:teuthology.orchestra.run.smithi033.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T00:55:01.576222Z", "last_refresh": "2024-09-17T00:58:34.030725Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T00:54:55.903476Z", "last_refresh": "2024-09-17T00:58:34.030794Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T00:54:58.805315Z", "last_refresh": "2024-09-17T00:58:34.030858Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T01:01:23.342599Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T01:01:23.335227Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T00:54:54.644049Z", "last_refresh": "2024-09-17T00:58:34.030651Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi033:172.21.15.33=smithi033", "smithi100:172.21.15.100=smithi100"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T00:55:43.948307Z", "last_refresh": "2024-09-17T00:58:34.030509Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T00:55:00.041575Z", "last_refresh": "2024-09-17T00:58:34.030922Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T00:58:34.031069Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T00:54:57.172843Z", "last_refresh": "2024-09-17T00:58:34.031004Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:boot 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-17T01:01:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:31 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:32.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:32.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:boot 2024-09-17T01:01:32.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:01:32.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:01:32.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:01:32.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: pgmap v217: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-17T01:01:32.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:31 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:32.046 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-17T01:01:33.047 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T01:01:33.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:33 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:33 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:34 smithi100 ceph-mon[25838]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-09-17T01:01:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:34 smithi033 ceph-mon[23750]: pgmap v218: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-09-17T01:01:36.027 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:36.044 INFO:teuthology.orchestra.run.smithi033.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T00:55:01.576222Z", "last_refresh": "2024-09-17T00:58:34.030725Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T00:54:55.903476Z", "last_refresh": "2024-09-17T00:58:34.030794Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T00:54:58.805315Z", "last_refresh": "2024-09-17T00:58:34.030858Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T01:01:23.342599Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T01:01:23.335227Z", "last_refresh": "2024-09-17T01:01:32.203998Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T00:54:54.644049Z", "last_refresh": "2024-09-17T00:58:34.030651Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi033:172.21.15.33=smithi033", "smithi100:172.21.15.100=smithi100"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T00:55:43.948307Z", "last_refresh": "2024-09-17T00:58:34.030509Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T00:55:00.041575Z", "last_refresh": "2024-09-17T00:58:34.030922Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T00:58:34.031069Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T00:54:57.172843Z", "last_refresh": "2024-09-17T00:58:34.031004Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T01:01:36.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:36 smithi033 ceph-mon[23750]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-17T01:01:36.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:36 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:36 smithi100 ceph-mon[25838]: pgmap v219: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-17T01:01:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:36 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:36.790 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-17T01:01:37.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:37 smithi100 ceph-mon[25838]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:37.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:37 smithi033 ceph-mon[23750]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:37.791 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T01:01:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T01:01:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:38.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:38.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:38.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:38 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: pgmap v220: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:38 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:39.787 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:01:39.787 INFO:teuthology.orchestra.run.smithi033.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T00:55:01.576222Z", "last_refresh": "2024-09-17T01:01:37.368038Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T00:54:55.903476Z", "last_refresh": "2024-09-17T01:01:32.203312Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-17T00:54:58.805315Z", "last_refresh": "2024-09-17T01:01:37.368170Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-17T01:01:23.342599Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T01:01:23.335227Z", "last_refresh": "2024-09-17T01:01:32.203998Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T00:54:54.644049Z", "last_refresh": "2024-09-17T01:01:32.203463Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi033:172.21.15.33=smithi033", "smithi100:172.21.15.100=smithi100"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T00:55:43.948307Z", "last_refresh": "2024-09-17T01:01:32.203547Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T00:55:00.041575Z", "last_refresh": "2024-09-17T01:01:32.203627Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-17T01:01:32.203701Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T00:54:57.172843Z", "last_refresh": "2024-09-17T01:01:37.368296Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T01:01:40.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:40 smithi033 ceph-mon[23750]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T01:01:40.425 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-17T01:01:40.426 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:01:40.437 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:01:40.437 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-17T01:01:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:40 smithi100 ceph-mon[25838]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-17T01:01:41.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:41 smithi033 ceph-mon[23750]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:41.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:41 smithi100 ceph-mon[25838]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:01:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:42 smithi100 ceph-mon[25838]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 682 B/s wr, 1 op/s 2024-09-17T01:01:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:42 smithi033 ceph-mon[23750]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 682 B/s wr, 1 op/s 2024-09-17T01:01:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:43 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T01:01:43.610 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:43 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-17T01:01:44.275 INFO:teuthology.orchestra.run.smithi033.stdout:NFS Cluster Created Successfully 2024-09-17T01:01:44.481 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:44 smithi033 ceph-mon[23750]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:44.481 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:44 smithi033 ceph-mon[23750]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2024-09-17T01:01:44.481 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:44 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T01:01:44.481 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:44 smithi033 ceph-mon[23750]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T01:01:44.481 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:44 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T01:01:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:44 smithi100 ceph-mon[25838]: from='client.14498 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:44 smithi100 ceph-mon[25838]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 85 B/s wr, 0 op/s 2024-09-17T01:01:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:44 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-17T01:01:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:44 smithi100 ceph-mon[25838]: osdmap e65: 8 total, 8 up, 8 in 2024-09-17T01:01:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:44 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-17T01:01:44.935 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-17T01:01:45.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T01:01:45.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T01:01:45.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: Saving service nfs.foo spec with placement count:2 2024-09-17T01:01:45.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:45.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:45.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:01:45.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:01:45.528 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:01:45.528 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: osdmap e66: 8 total, 8 up, 8 in 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: Saving service nfs.foo spec with placement count:2 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:01:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:01:45.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:46.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:46 smithi033 ceph-mon[23750]: Rados config object exists: conf-nfs.foo 2024-09-17T01:01:46.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:46 smithi033 ceph-mon[23750]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh-rgw 2024-09-17T01:01:46.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:46 smithi033 ceph-mon[23750]: Deploying daemon nfs.foo.0.0.smithi033.ttskqh on smithi033 2024-09-17T01:01:46.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:46 smithi033 ceph-mon[23750]: pgmap v226: 97 pgs: 11 creating+peering, 14 unknown, 72 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-09-17T01:01:46.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:46 smithi033 ceph-mon[23750]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T01:01:46.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:46 smithi033 ceph-mon[23750]: mgrmap e20: smithi033.illmog(active, since 5m), standbys: smithi100.qrwkie 2024-09-17T01:01:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:46 smithi100 ceph-mon[25838]: Rados config object exists: conf-nfs.foo 2024-09-17T01:01:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:46 smithi100 ceph-mon[25838]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh-rgw 2024-09-17T01:01:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:46 smithi100 ceph-mon[25838]: Deploying daemon nfs.foo.0.0.smithi033.ttskqh on smithi033 2024-09-17T01:01:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:46 smithi100 ceph-mon[25838]: pgmap v226: 97 pgs: 11 creating+peering, 14 unknown, 72 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 255 B/s wr, 0 op/s 2024-09-17T01:01:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:46 smithi100 ceph-mon[25838]: osdmap e67: 8 total, 8 up, 8 in 2024-09-17T01:01:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:46 smithi100 ceph-mon[25838]: mgrmap e20: smithi033.illmog(active, since 5m), standbys: smithi100.qrwkie 2024-09-17T01:01:48.424 INFO:teuthology.orchestra.run.smithi033.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-17T01:01:48.425 INFO:teuthology.orchestra.run.smithi033.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-17T01:01:48.425 INFO:teuthology.orchestra.run.smithi033.stderr:Error EINVAL: invalid command 2024-09-17T01:01:48.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:48 smithi033 ceph-mon[23750]: pgmap v228: 97 pgs: 11 creating+peering, 5 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T01:01:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:48 smithi100 ceph-mon[25838]: pgmap v228: 97 pgs: 11 creating+peering, 5 unknown, 81 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T01:01:48.784 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:01:48.784 INFO:teuthology.orchestra.run.smithi033.stdout: "bind": "/fake", 2024-09-17T01:01:48.784 INFO:teuthology.orchestra.run.smithi033.stdout: "fs": "foofs", 2024-09-17T01:01:48.785 INFO:teuthology.orchestra.run.smithi033.stdout: "path": "/", 2024-09-17T01:01:48.785 INFO:teuthology.orchestra.run.smithi033.stdout: "cluster": "foo", 2024-09-17T01:01:48.785 INFO:teuthology.orchestra.run.smithi033.stdout: "mode": "RW" 2024-09-17T01:01:48.785 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:01:49.430 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:49.742 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T01:01:49.743 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T01:01:49.743 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:49 smithi033 ceph-mon[23750]: pgmap v229: 97 pgs: 6 creating+peering, 91 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T01:01:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp 2024-09-17T01:01:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:01:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='client.14528 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-17T01:01:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:49 smithi100 ceph-mon[25838]: pgmap v229: 97 pgs: 6 creating+peering, 91 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-17T01:01:52.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:51 smithi033 ceph-mon[23750]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-17T01:01:52.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:51 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:01:52.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:51 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:01:52.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:51 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:01:52.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:51 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:01:52.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:51 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:52.183 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:51 smithi100 ceph-mon[25838]: pgmap v230: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-17T01:01:52.183 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:51 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:01:52.184 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:51 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:01:52.184 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:51 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:01:52.184 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:51 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-17T01:01:52.184 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:51 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:53.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:52 smithi100 ceph-mon[25838]: Rados config object exists: conf-nfs.foo 2024-09-17T01:01:53.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:52 smithi100 ceph-mon[25838]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp-rgw 2024-09-17T01:01:53.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:52 smithi100 ceph-mon[25838]: Deploying daemon nfs.foo.1.0.smithi100.mvigkp on smithi100 2024-09-17T01:01:53.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:52 smithi100 ceph-mon[25838]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:52 smithi033 ceph-mon[23750]: Rados config object exists: conf-nfs.foo 2024-09-17T01:01:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:52 smithi033 ceph-mon[23750]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp-rgw 2024-09-17T01:01:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:52 smithi033 ceph-mon[23750]: Deploying daemon nfs.foo.1.0.smithi100.mvigkp on smithi100 2024-09-17T01:01:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:52 smithi033 ceph-mon[23750]: from='client.14536 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:54.161 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:53 smithi100 ceph-mon[25838]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:54.161 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:53 smithi100 ceph-mon[25838]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-17T01:01:54.208 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:53 smithi033 ceph-mon[23750]: from='client.14550 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:54.208 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:53 smithi033 ceph-mon[23750]: pgmap v231: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.2 KiB/s wr, 2 op/s 2024-09-17T01:01:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:54 smithi100 ceph-mon[25838]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:54 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:54 smithi033 ceph-mon[23750]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:01:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:01:55.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:54 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:01:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:55 smithi100 ceph-mon[25838]: pgmap v232: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T01:01:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:55 smithi033 ceph-mon[23750]: pgmap v232: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T01:01:56.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:57.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:56 smithi100 ceph-mon[25838]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:57.303 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:56 smithi033 ceph-mon[23750]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:58.143 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:58 smithi033 ceph-mon[23750]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:58.143 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:58 smithi033 ceph-mon[23750]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-09-17T01:01:58.143 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:58 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:58 smithi100 ceph-mon[25838]: from='client.14574 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:58 smithi100 ceph-mon[25838]: pgmap v233: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-09-17T01:01:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:58 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:01:59.223 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:01:59 smithi033 ceph-mon[23750]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:01:59.473 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo ?:2049 2/2 2s ago 15s count:2 2024-09-17T01:01:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:01:59 smithi100 ceph-mon[25838]: from='client.14578 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:00.065 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T01:02:00.074 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:02:00.074 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-17T01:02:00.102 INFO:teuthology.orchestra.run.smithi033.stderr:+ mkdir /mnt/foo 2024-09-17T01:02:00.112 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-17T01:02:00.177 INFO:teuthology.orchestra.run.smithi033.stderr:++ hostname 2024-09-17T01:02:00.177 INFO:teuthology.orchestra.run.smithi033.stderr:+ mount -t nfs smithi033:/fake /mnt/foo -o sync 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.319 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:00 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.623 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-17T01:02:00.689 INFO:teuthology.orchestra.run.smithi033.stderr:+ echo test 2024-09-17T01:02:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: pgmap v234: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T01:02:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:02:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:02:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:02:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:00 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:00.734 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-17T01:02:00.799 INFO:teuthology.orchestra.run.smithi033.stderr:+ sync 2024-09-17T01:02:01.038 INFO:teuthology.run_tasks:Running task parallel... 2024-09-17T01:02:01.053 INFO:teuthology.task.parallel:starting parallel... 2024-09-17T01:02:01.053 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T01:02:01.054 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T01:02:01.054 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:02:01.055 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-17T01:02:01.055 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-17T01:02:01.055 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-17T01:02:01.056 INFO:teuthology.task.exec:Executing custom commands... 2024-09-17T01:02:01.056 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:02:01.056 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-17T01:02:01.129 INFO:teuthology.orchestra.run.smithi033.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T01:02:01.129 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:02:01.129 INFO:teuthology.orchestra.run.smithi033.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-17T01:02:01.137 INFO:teuthology.orchestra.run.smithi033.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-17T01:02:01.237 INFO:teuthology.orchestra.run.smithi033.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T01:02:01.238 INFO:teuthology.orchestra.run.smithi033.stdout:releasing clients 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:02:01.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:02:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='client.14582 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]: dispatch 2024-09-17T01:02:01.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-09-17T01:02:01.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-17T01:02:01.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-09-17T01:02:01.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:02:01.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:02:02.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 142 30.70 MB/sec warmup 1 sec latency 38.486 ms 2024-09-17T01:02:02.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:02 smithi033 ceph-mon[23750]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T01:02:02.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-09-17T01:02:02.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-17T01:02:02.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-17T01:02:02.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:02 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-17T01:02:02.432 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:02 smithi033 ceph-mon[23750]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T01:02:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:02 smithi100 ceph-mon[25838]: pgmap v235: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.1 KiB/s wr, 6 op/s 2024-09-17T01:02:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.f", "id": [7, 2]}]': finished 2024-09-17T01:02:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-09-17T01:02:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-17T01:02:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:02 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-17T01:02:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:02 smithi100 ceph-mon[25838]: osdmap e68: 8 total, 8 up, 8 in 2024-09-17T01:02:03.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 280 26.11 MB/sec warmup 2 sec latency 30.533 ms 2024-09-17T01:02:03.456 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:03 smithi033 ceph-mon[23750]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T01:02:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:03 smithi100 ceph-mon[25838]: osdmap e69: 8 total, 8 up, 8 in 2024-09-17T01:02:04.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 447 26.10 MB/sec warmup 3 sec latency 23.752 ms 2024-09-17T01:02:04.478 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:04 smithi033 ceph-mon[23750]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 511 B/s wr, 0 op/s 2024-09-17T01:02:04.621 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-17T01:02:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:04 smithi100 ceph-mon[25838]: pgmap v238: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 511 B/s wr, 0 op/s 2024-09-17T01:02:05.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 565 23.94 MB/sec warmup 4 sec latency 55.184 ms 2024-09-17T01:02:06.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 851 21.57 MB/sec warmup 5 sec latency 31.152 ms 2024-09-17T01:02:06.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:06 smithi033 ceph-mon[23750]: pgmap v239: 97 pgs: 97 active+clean; 55 MiB data, 94 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 7.0 MiB/s wr, 278 op/s 2024-09-17T01:02:06.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:06 smithi100 ceph-mon[25838]: pgmap v239: 97 pgs: 97 active+clean; 55 MiB data, 94 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 7.0 MiB/s wr, 278 op/s 2024-09-17T01:02:07.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 1241 19.24 MB/sec warmup 6 sec latency 39.357 ms 2024-09-17T01:02:08.238 INFO:teuthology.orchestra.run.smithi033.stdout: 5 2123 19.05 MB/sec warmup 7 sec latency 21.759 ms 2024-09-17T01:02:08.372 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:08 smithi033 ceph-mon[23750]: pgmap v240: 97 pgs: 97 active+clean; 66 MiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.4 MiB/s wr, 310 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:08 smithi100 ceph-mon[25838]: pgmap v240: 97 pgs: 97 active+clean; 66 MiB data, 99 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.4 MiB/s wr, 310 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:08.852 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-17T01:02:09.248 INFO:teuthology.orchestra.run.smithi033.stdout: 5 2954 20.89 MB/sec warmup 8 sec latency 52.102 ms 2024-09-17T01:02:10.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 3621 21.41 MB/sec warmup 9 sec latency 22.808 ms 2024-09-17T01:02:10.386 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:10 smithi033 ceph-mon[23750]: pgmap v241: 97 pgs: 97 active+clean; 96 MiB data, 169 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 540 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:10 smithi100 ceph-mon[25838]: pgmap v241: 97 pgs: 97 active+clean; 96 MiB data, 169 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 13 MiB/s wr, 540 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:11.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 3853 19.89 MB/sec warmup 10 sec latency 55.102 ms 2024-09-17T01:02:12.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 4323 18.37 MB/sec warmup 11 sec latency 62.935 ms 2024-09-17T01:02:12.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:12 smithi033 ceph-mon[23750]: pgmap v242: 97 pgs: 97 active+clean; 115 MiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 13 MiB/s wr, 638 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:12 smithi100 ceph-mon[25838]: pgmap v242: 97 pgs: 97 active+clean; 115 MiB data, 182 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 13 MiB/s wr, 638 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:13.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 4974 18.79 MB/sec warmup 12 sec latency 32.422 ms 2024-09-17T01:02:13.913 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:13 smithi033 ceph-mon[23750]: pgmap v243: 97 pgs: 97 active+clean; 117 MiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 12 MiB/s wr, 596 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:14.051 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph mgr module enable nfs --force' 2024-09-17T01:02:14.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:13 smithi100 ceph-mon[25838]: pgmap v243: 97 pgs: 97 active+clean; 117 MiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 12 MiB/s wr, 596 op/s; 2 B/s, 0 objects/s recovering 2024-09-17T01:02:14.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 5638 17.58 MB/sec warmup 13 sec latency 28.793 ms 2024-09-17T01:02:15.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 6021 16.96 MB/sec warmup 14 sec latency 43.304 ms 2024-09-17T01:02:15.972 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:15 smithi033 ceph-mon[23750]: pgmap v244: 97 pgs: 97 active+clean; 132 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 14 MiB/s wr, 887 op/s; 1 B/s, 0 objects/s recovering 2024-09-17T01:02:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:15 smithi100 ceph-mon[25838]: pgmap v244: 97 pgs: 97 active+clean; 132 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 14 MiB/s wr, 887 op/s; 1 B/s, 0 objects/s recovering 2024-09-17T01:02:16.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 6566 17.54 MB/sec warmup 15 sec latency 34.241 ms 2024-09-17T01:02:17.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 7130 17.99 MB/sec warmup 16 sec latency 55.849 ms 2024-09-17T01:02:17.970 INFO:teuthology.orchestra.run.smithi033.stderr:module 'nfs' is already enabled 2024-09-17T01:02:18.061 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:17 smithi033 ceph-mon[23750]: pgmap v245: 97 pgs: 97 active+clean; 131 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 9.9 MiB/s wr, 717 op/s; 1 B/s, 0 objects/s recovering 2024-09-17T01:02:18.062 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:17 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3080566648' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T01:02:18.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:17 smithi100 ceph-mon[25838]: pgmap v245: 97 pgs: 97 active+clean; 131 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 9.9 MiB/s wr, 717 op/s; 1 B/s, 0 objects/s recovering 2024-09-17T01:02:18.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:17 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3080566648' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-17T01:02:18.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 7324 17.29 MB/sec warmup 17 sec latency 69.283 ms 2024-09-17T01:02:19.005 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-17T01:02:19.080 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:18 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3080566648' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T01:02:19.081 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:18 smithi033 ceph-mon[23750]: mgrmap e21: smithi033.illmog(active, since 6m), standbys: smithi100.qrwkie 2024-09-17T01:02:19.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 7460 16.40 MB/sec warmup 18 sec latency 77.339 ms 2024-09-17T01:02:19.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:18 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3080566648' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-17T01:02:19.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:18 smithi100 ceph-mon[25838]: mgrmap e21: smithi033.illmog(active, since 6m), standbys: smithi100.qrwkie 2024-09-17T01:02:20.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:19 smithi033 ceph-mon[23750]: pgmap v246: 97 pgs: 97 active+clean; 132 MiB data, 306 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 10 MiB/s wr, 820 op/s 2024-09-17T01:02:20.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 7916 15.70 MB/sec warmup 19 sec latency 55.310 ms 2024-09-17T01:02:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:19 smithi100 ceph-mon[25838]: pgmap v246: 97 pgs: 97 active+clean; 132 MiB data, 306 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 10 MiB/s wr, 820 op/s 2024-09-17T01:02:21.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 8147 15.14 MB/sec warmup 20 sec latency 81.150 ms 2024-09-17T01:02:22.136 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:21 smithi033 ceph-mon[23750]: pgmap v247: 97 pgs: 97 active+clean; 141 MiB data, 332 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.8 MiB/s wr, 752 op/s 2024-09-17T01:02:22.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 8600 15.30 MB/sec warmup 21 sec latency 70.067 ms 2024-09-17T01:02:22.367 INFO:teuthology.orchestra.run.smithi033.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:02:22.418 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:21 smithi100 ceph-mon[25838]: pgmap v247: 97 pgs: 97 active+clean; 141 MiB data, 332 MiB used, 715 GiB / 715 GiB avail; 9.0 KiB/s rd, 8.8 MiB/s wr, 752 op/s 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:23.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:22 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:23.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 9169 14.77 MB/sec warmup 22 sec latency 51.980 ms 2024-09-17T01:02:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:02:23.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:02:23.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:02:23.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:23.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:22 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:23.753 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-17T01:02:23.754 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:02:23.755 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-17T01:02:24.095 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:23 smithi033 ceph-mon[23750]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:02:24.095 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:23 smithi033 ceph-mon[23750]: pgmap v248: 97 pgs: 97 active+clean; 142 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.9 MiB/s wr, 607 op/s 2024-09-17T01:02:24.095 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:23 smithi033 ceph-mon[23750]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:02:24.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 9479 14.38 MB/sec warmup 23 sec latency 46.052 ms 2024-09-17T01:02:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:23 smithi100 ceph-mon[25838]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:02:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:23 smithi100 ceph-mon[25838]: pgmap v248: 97 pgs: 97 active+clean; 142 MiB data, 337 MiB used, 715 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.9 MiB/s wr, 607 op/s 2024-09-17T01:02:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:23 smithi100 ceph-mon[25838]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-17T01:02:25.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 9748 14.15 MB/sec warmup 24 sec latency 55.203 ms 2024-09-17T01:02:26.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:25 smithi033 ceph-mon[23750]: pgmap v249: 97 pgs: 97 active+clean; 138 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 797 op/s 2024-09-17T01:02:26.249 INFO:teuthology.orchestra.run.smithi033.stdout: 5 10312 14.53 MB/sec warmup 25 sec latency 32.515 ms 2024-09-17T01:02:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:25 smithi100 ceph-mon[25838]: pgmap v249: 97 pgs: 97 active+clean; 138 MiB data, 419 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 797 op/s 2024-09-17T01:02:26.896 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:02:27.251 INFO:teuthology.orchestra.run.smithi033.stdout: 5 10767 14.82 MB/sec warmup 26 sec latency 40.041 ms 2024-09-17T01:02:28.001 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:02:28.001 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (5m) 28s ago 6m 22.8M - ba2b418f427c a6247209f546 2024-09-17T01:02:28.001 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (6m) 28s ago 6m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:02:28.001 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (6m) 30s ago 6m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:02:28.001 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (5m) 28s ago 6m 52.7M - 8.3.5 dad864ee21e9 e7569df2559d 2024-09-17T01:02:28.001 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (61s) 28s ago 60s 12.4M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (58s) 30s ago 57s 12.1M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:9283 running (8m) 28s ago 8m 460M - 17.2.0 e1d6a67b021e 4df4282576a9 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (6m) 30s ago 6m 414M - 17.2.0 e1d6a67b021e 4c4fbb94047d 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (8m) 28s ago 8m 46.8M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (6m) 30s ago 6m 39.1M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (40s) 28s ago 39s 51.2M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (34s) 30s ago 33s 50.8M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (6m) 28s ago 6m 11.9M - 1dbe0e931976 65382f92c0e1 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (5m) 30s ago 5m 17.6M - 1dbe0e931976 7fb37007ed80 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (5m) 28s ago 5m 42.7M 3217M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (4m) 28s ago 4m 43.2M 3217M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (4m) 28s ago 4m 40.3M 3217M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (4m) 28s ago 4m 40.8M 3217M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:02:28.002 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (3m) 30s ago 3m 39.9M 3983M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:02:28.003 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (3m) 30s ago 3m 41.9M 3983M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:02:28.003 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (2m) 30s ago 2m 40.0M 3983M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:02:28.003 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (2m) 30s ago 2m 41.9M 3983M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:02:28.003 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (5m) 28s ago 5m 63.8M - 514e6a882f6e 3afe8f20ef9f 2024-09-17T01:02:28.167 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:27 smithi033 ceph-mon[23750]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:28.167 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:27 smithi033 ceph-mon[23750]: pgmap v250: 97 pgs: 97 active+clean; 137 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 450 op/s 2024-09-17T01:02:28.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 11000 14.47 MB/sec warmup 27 sec latency 47.441 ms 2024-09-17T01:02:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:27 smithi100 ceph-mon[25838]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:27 smithi100 ceph-mon[25838]: pgmap v250: 97 pgs: 97 active+clean; 137 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 450 op/s 2024-09-17T01:02:28.682 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:02:28.682 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:02:28.683 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:02:28.684 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:02:28.684 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T01:02:28.684 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:02:28.684 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:02:29.173 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:29 smithi033 ceph-mon[23750]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:29.173 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:29 smithi033 ceph-mon[23750]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:29.173 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:29 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1188283568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "", 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f image" 2024-09-17T01:02:29.238 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:02:29.251 INFO:teuthology.orchestra.run.smithi033.stdout: 5 11360 14.03 MB/sec warmup 28 sec latency 32.337 ms 2024-09-17T01:02:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:29 smithi100 ceph-mon[25838]: from='client.24393 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:29 smithi100 ceph-mon[25838]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:29 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1188283568' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:02:29.885 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:02:30.141 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:30 smithi033 ceph-mon[23750]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-09-17T01:02:30.141 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:30 smithi033 ceph-mon[23750]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:30.141 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:30 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3519856533' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:02:30.251 INFO:teuthology.orchestra.run.smithi033.stdout: 5 11710 13.74 MB/sec warmup 29 sec latency 34.981 ms 2024-09-17T01:02:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:30 smithi100 ceph-mon[25838]: pgmap v251: 97 pgs: 97 active+clean; 147 MiB data, 409 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 6.1 MiB/s wr, 532 op/s 2024-09-17T01:02:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:30 smithi100 ceph-mon[25838]: from='client.24403 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:02:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:30 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3519856533' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:02:31.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 12072 13.71 MB/sec warmup 30 sec latency 38.624 ms 2024-09-17T01:02:32.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 12487 13.50 MB/sec warmup 31 sec latency 62.011 ms 2024-09-17T01:02:32.273 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:32 smithi033 ceph-mon[23750]: pgmap v252: 97 pgs: 97 active+clean; 154 MiB data, 416 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 517 op/s 2024-09-17T01:02:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:32 smithi100 ceph-mon[25838]: pgmap v252: 97 pgs: 97 active+clean; 154 MiB data, 416 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 517 op/s 2024-09-17T01:02:33.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 13184 13.49 MB/sec warmup 32 sec latency 19.699 ms 2024-09-17T01:02:34.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:33 smithi033 ceph-mon[23750]: pgmap v253: 97 pgs: 97 active+clean; 153 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 440 op/s 2024-09-17T01:02:34.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:33 smithi100 ceph-mon[25838]: pgmap v253: 97 pgs: 97 active+clean; 153 MiB data, 414 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 440 op/s 2024-09-17T01:02:34.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 13688 13.75 MB/sec warmup 33 sec latency 41.002 ms 2024-09-17T01:02:35.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 14100 13.83 MB/sec warmup 34 sec latency 63.308 ms 2024-09-17T01:02:36.065 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:35 smithi033 ceph-mon[23750]: pgmap v254: 97 pgs: 97 active+clean; 155 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 709 op/s 2024-09-17T01:02:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:35 smithi100 ceph-mon[25838]: pgmap v254: 97 pgs: 97 active+clean; 155 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 6.7 MiB/s wr, 709 op/s 2024-09-17T01:02:36.252 INFO:teuthology.orchestra.run.smithi033.stdout: 5 14440 13.84 MB/sec warmup 35 sec latency 34.728 ms 2024-09-17T01:02:37.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 15073 13.66 MB/sec warmup 36 sec latency 23.578 ms 2024-09-17T01:02:38.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:37 smithi033 ceph-mon[23750]: pgmap v255: 97 pgs: 97 active+clean; 157 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 515 op/s 2024-09-17T01:02:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:37 smithi100 ceph-mon[25838]: pgmap v255: 97 pgs: 97 active+clean; 157 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 515 op/s 2024-09-17T01:02:38.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 15341 13.46 MB/sec warmup 37 sec latency 54.542 ms 2024-09-17T01:02:39.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 15672 13.27 MB/sec warmup 38 sec latency 74.805 ms 2024-09-17T01:02:40.152 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:39 smithi033 ceph-mon[23750]: pgmap v256: 97 pgs: 97 active+clean; 168 MiB data, 537 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.8 MiB/s wr, 641 op/s 2024-09-17T01:02:40.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:39 smithi100 ceph-mon[25838]: pgmap v256: 97 pgs: 97 active+clean; 168 MiB data, 537 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 6.8 MiB/s wr, 641 op/s 2024-09-17T01:02:40.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 16058 13.19 MB/sec warmup 39 sec latency 52.572 ms 2024-09-17T01:02:41.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 16463 12.96 MB/sec warmup 40 sec latency 66.104 ms 2024-09-17T01:02:42.187 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:41 smithi033 ceph-mon[23750]: pgmap v257: 97 pgs: 97 active+clean; 168 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 641 op/s 2024-09-17T01:02:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:41 smithi100 ceph-mon[25838]: pgmap v257: 97 pgs: 97 active+clean; 168 MiB data, 556 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.4 MiB/s wr, 641 op/s 2024-09-17T01:02:42.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 17023 13.17 MB/sec warmup 41 sec latency 41.506 ms 2024-09-17T01:02:43.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 17803 13.73 MB/sec warmup 42 sec latency 19.469 ms 2024-09-17T01:02:43.952 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:43 smithi033 ceph-mon[23750]: pgmap v258: 97 pgs: 97 active+clean; 167 MiB data, 573 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-09-17T01:02:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:43 smithi100 ceph-mon[25838]: pgmap v258: 97 pgs: 97 active+clean; 167 MiB data, 573 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.5 MiB/s wr, 541 op/s 2024-09-17T01:02:44.256 INFO:teuthology.orchestra.run.smithi033.stdout: 5 18062 13.53 MB/sec warmup 43 sec latency 47.137 ms 2024-09-17T01:02:45.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 18383 13.27 MB/sec warmup 44 sec latency 39.125 ms 2024-09-17T01:02:45.963 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:45 smithi033 ceph-mon[23750]: pgmap v259: 97 pgs: 97 active+clean; 177 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 791 op/s 2024-09-17T01:02:46.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:45 smithi100 ceph-mon[25838]: pgmap v259: 97 pgs: 97 active+clean; 177 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 791 op/s 2024-09-17T01:02:46.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 19058 13.31 MB/sec warmup 45 sec latency 14.748 ms 2024-09-17T01:02:47.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 19726 13.29 MB/sec warmup 46 sec latency 28.614 ms 2024-09-17T01:02:47.975 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:47 smithi033 ceph-mon[23750]: pgmap v260: 97 pgs: 97 active+clean; 179 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 526 op/s 2024-09-17T01:02:48.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:47 smithi100 ceph-mon[25838]: pgmap v260: 97 pgs: 97 active+clean; 179 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 526 op/s 2024-09-17T01:02:48.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 20413 13.37 MB/sec warmup 47 sec latency 31.148 ms 2024-09-17T01:02:49.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 20962 13.60 MB/sec warmup 48 sec latency 22.415 ms 2024-09-17T01:02:50.064 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:49 smithi033 ceph-mon[23750]: pgmap v261: 97 pgs: 97 active+clean; 181 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.5 MiB/s wr, 733 op/s 2024-09-17T01:02:50.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:49 smithi100 ceph-mon[25838]: pgmap v261: 97 pgs: 97 active+clean; 181 MiB data, 697 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.5 MiB/s wr, 733 op/s 2024-09-17T01:02:50.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 21416 13.73 MB/sec warmup 49 sec latency 39.232 ms 2024-09-17T01:02:51.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 21693 13.55 MB/sec warmup 50 sec latency 47.521 ms 2024-09-17T01:02:52.103 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:51 smithi033 ceph-mon[23750]: pgmap v262: 97 pgs: 97 active+clean; 182 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 699 op/s 2024-09-17T01:02:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:51 smithi100 ceph-mon[25838]: pgmap v262: 97 pgs: 97 active+clean; 182 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.0 MiB/s wr, 699 op/s 2024-09-17T01:02:52.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 21946 13.33 MB/sec warmup 51 sec latency 59.273 ms 2024-09-17T01:02:53.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 22308 13.16 MB/sec warmup 52 sec latency 44.155 ms 2024-09-17T01:02:53.876 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:53 smithi033 ceph-mon[23750]: pgmap v263: 97 pgs: 97 active+clean; 183 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 621 op/s 2024-09-17T01:02:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:53 smithi100 ceph-mon[25838]: pgmap v263: 97 pgs: 97 active+clean; 183 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.3 MiB/s wr, 621 op/s 2024-09-17T01:02:54.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 22708 13.12 MB/sec warmup 53 sec latency 58.337 ms 2024-09-17T01:02:55.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 23102 13.08 MB/sec warmup 54 sec latency 44.045 ms 2024-09-17T01:02:55.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:55.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:55 smithi033 ceph-mon[23750]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (unknown) 2024-09-17T01:02:55.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:55 smithi033 ceph-mon[23750]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:02:55.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:02:55.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:55 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:55.433 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:55 smithi033 ceph-mon[23750]: Upgrade: Need to upgrade myself (mgr.smithi033.illmog) 2024-09-17T01:02:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:55 smithi100 ceph-mon[25838]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (unknown) 2024-09-17T01:02:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:55 smithi100 ceph-mon[25838]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-17T01:02:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:02:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:55 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:02:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:55 smithi100 ceph-mon[25838]: Upgrade: Need to upgrade myself (mgr.smithi033.illmog) 2024-09-17T01:02:56.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 23578 12.92 MB/sec warmup 55 sec latency 75.090 ms 2024-09-17T01:02:56.461 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:56 smithi033 ceph-mon[23750]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 on smithi100 2024-09-17T01:02:56.461 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:56 smithi033 ceph-mon[23750]: pgmap v264: 97 pgs: 97 active+clean; 194 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 797 op/s 2024-09-17T01:02:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:56 smithi100 ceph-mon[25838]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 on smithi100 2024-09-17T01:02:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:56 smithi100 ceph-mon[25838]: pgmap v264: 97 pgs: 97 active+clean; 194 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.4 MiB/s wr, 797 op/s 2024-09-17T01:02:57.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 23950 12.92 MB/sec warmup 56 sec latency 43.000 ms 2024-09-17T01:02:58.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 24284 12.96 MB/sec warmup 57 sec latency 45.069 ms 2024-09-17T01:02:58.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:02:58 smithi033 ceph-mon[23750]: pgmap v265: 97 pgs: 97 active+clean; 193 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 565 op/s 2024-09-17T01:02:58.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:02:58 smithi100 ceph-mon[25838]: pgmap v265: 97 pgs: 97 active+clean; 193 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 5.8 MiB/s wr, 565 op/s 2024-09-17T01:02:59.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 24593 12.93 MB/sec warmup 58 sec latency 49.065 ms 2024-09-17T01:03:00.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 24970 12.98 MB/sec warmup 59 sec latency 157.798 ms 2024-09-17T01:03:00.269 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:03:00.339 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:00 smithi033 ceph-mon[23750]: pgmap v266: 97 pgs: 97 active+clean; 192 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 667 op/s 2024-09-17T01:03:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:00 smithi100 ceph-mon[25838]: pgmap v266: 97 pgs: 97 active+clean; 192 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.6 MiB/s wr, 667 op/s 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (6m) 61s ago 7m 22.8M - ba2b418f427c a6247209f546 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (7m) 61s ago 7m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (6m) 64s ago 6m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (5m) 61s ago 7m 52.7M - 8.3.5 dad864ee21e9 e7569df2559d 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (94s) 61s ago 93s 12.4M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (91s) 64s ago 90s 12.1M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:03:01.123 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:9283 running (8m) 61s ago 8m 460M - 17.2.0 e1d6a67b021e 4df4282576a9 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (6m) 64s ago 6m 414M - 17.2.0 e1d6a67b021e 4c4fbb94047d 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (8m) 61s ago 8m 46.8M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (6m) 64s ago 6m 39.1M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (73s) 61s ago 72s 51.2M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (67s) 64s ago 66s 50.8M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (7m) 61s ago 7m 11.9M - 1dbe0e931976 65382f92c0e1 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (6m) 64s ago 6m 17.6M - 1dbe0e931976 7fb37007ed80 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (5m) 61s ago 5m 42.7M 3217M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (5m) 61s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (4m) 61s ago 4m 40.3M 3217M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (4m) 61s ago 4m 40.8M 3217M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (4m) 64s ago 4m 39.9M 3983M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:03:01.124 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (3m) 64s ago 3m 41.9M 3983M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:03:01.125 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (3m) 64s ago 3m 40.0M 3983M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:03:01.125 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (2m) 64s ago 2m 41.9M 3983M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:03:01.125 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (6m) 61s ago 6m 63.8M - 514e6a882f6e 3afe8f20ef9f 2024-09-17T01:03:01.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 25247 12.89 MB/sec warmup 60 sec latency 187.203 ms 2024-09-17T01:03:01.382 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:01 smithi033 ceph-mon[23750]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:01.382 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:03:01.382 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:01 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:03:01.670 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:03:01.670 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:03:01.671 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T01:03:01.672 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:03:01.672 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:03:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:01 smithi100 ceph-mon[25838]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:03:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:01 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "0/23 daemons upgraded", 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 image on host smithi100" 2024-09-17T01:03:02.089 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:03:02.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 25592 12.73 MB/sec warmup 61 sec latency 55.216 ms 2024-09-17T01:03:02.386 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:02 smithi033 ceph-mon[23750]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:02.386 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:02 smithi033 ceph-mon[23750]: pgmap v267: 97 pgs: 97 active+clean; 194 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.5 MiB/s wr, 482 op/s 2024-09-17T01:03:02.386 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:02 smithi033 ceph-mon[23750]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:02.386 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:02 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/303644873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:03:02.715 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:03:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:02 smithi100 ceph-mon[25838]: from='client.14632 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:02 smithi100 ceph-mon[25838]: pgmap v267: 97 pgs: 97 active+clean; 194 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.5 MiB/s wr, 482 op/s 2024-09-17T01:03:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:02 smithi100 ceph-mon[25838]: from='client.14636 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:02 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/303644873' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:03:03.253 INFO:teuthology.orchestra.run.smithi033.stdout: 5 26050 12.66 MB/sec warmup 62 sec latency 43.641 ms 2024-09-17T01:03:03.401 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:03 smithi033 ceph-mon[23750]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:03.401 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:03 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/199259618' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:03:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:03 smithi100 ceph-mon[25838]: from='client.14644 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:03 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/199259618' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:03:04.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 26603 12.73 MB/sec warmup 63 sec latency 44.183 ms 2024-09-17T01:03:04.411 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:04 smithi033 ceph-mon[23750]: pgmap v268: 97 pgs: 97 active+clean; 199 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-09-17T01:03:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:04 smithi100 ceph-mon[25838]: pgmap v268: 97 pgs: 97 active+clean; 199 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.1 MiB/s wr, 471 op/s 2024-09-17T01:03:05.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 27265 12.67 MB/sec warmup 64 sec latency 35.800 ms 2024-09-17T01:03:06.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 27650 12.68 MB/sec warmup 65 sec latency 43.358 ms 2024-09-17T01:03:06.474 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:06 smithi033 ceph-mon[23750]: pgmap v269: 97 pgs: 97 active+clean; 204 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 639 op/s 2024-09-17T01:03:06.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:06 smithi100 ceph-mon[25838]: pgmap v269: 97 pgs: 97 active+clean; 204 MiB data, 843 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.8 MiB/s wr, 639 op/s 2024-09-17T01:03:07.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 28285 12.90 MB/sec warmup 66 sec latency 97.057 ms 2024-09-17T01:03:08.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 28586 12.90 MB/sec warmup 67 sec latency 56.586 ms 2024-09-17T01:03:08.517 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:08 smithi033 ceph-mon[23750]: pgmap v270: 97 pgs: 97 active+clean; 203 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-09-17T01:03:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:08 smithi100 ceph-mon[25838]: pgmap v270: 97 pgs: 97 active+clean; 203 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-09-17T01:03:09.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 29049 12.82 MB/sec warmup 68 sec latency 77.443 ms 2024-09-17T01:03:10.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 29430 12.71 MB/sec warmup 69 sec latency 42.570 ms 2024-09-17T01:03:10.530 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:10 smithi033 ceph-mon[23750]: pgmap v271: 97 pgs: 97 active+clean; 208 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 663 op/s 2024-09-17T01:03:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:10 smithi100 ceph-mon[25838]: pgmap v271: 97 pgs: 97 active+clean; 208 MiB data, 915 MiB used, 714 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 663 op/s 2024-09-17T01:03:11.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 29952 12.73 MB/sec warmup 70 sec latency 65.565 ms 2024-09-17T01:03:12.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 30244 12.65 MB/sec warmup 71 sec latency 63.002 ms 2024-09-17T01:03:12.565 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:12 smithi033 ceph-mon[23750]: pgmap v272: 97 pgs: 97 active+clean; 215 MiB data, 919 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.4 MiB/s wr, 607 op/s 2024-09-17T01:03:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:12 smithi100 ceph-mon[25838]: pgmap v272: 97 pgs: 97 active+clean; 215 MiB data, 919 MiB used, 714 GiB / 715 GiB avail; 9.6 KiB/s rd, 6.4 MiB/s wr, 607 op/s 2024-09-17T01:03:13.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 30930 12.64 MB/sec warmup 72 sec latency 32.081 ms 2024-09-17T01:03:13.855 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:13 smithi033 ceph-mon[23750]: pgmap v273: 97 pgs: 97 active+clean; 216 MiB data, 919 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 588 op/s 2024-09-17T01:03:14.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:13 smithi100 ceph-mon[25838]: pgmap v273: 97 pgs: 97 active+clean; 216 MiB data, 919 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 6.1 MiB/s wr, 588 op/s 2024-09-17T01:03:14.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 31135 12.57 MB/sec warmup 73 sec latency 58.891 ms 2024-09-17T01:03:15.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 31509 12.66 MB/sec warmup 74 sec latency 44.512 ms 2024-09-17T01:03:16.167 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:15 smithi033 ceph-mon[23750]: pgmap v274: 97 pgs: 97 active+clean; 216 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 735 op/s 2024-09-17T01:03:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:15 smithi100 ceph-mon[25838]: pgmap v274: 97 pgs: 97 active+clean; 216 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 735 op/s 2024-09-17T01:03:16.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 31858 12.64 MB/sec warmup 75 sec latency 66.400 ms 2024-09-17T01:03:17.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 32120 12.59 MB/sec warmup 76 sec latency 85.679 ms 2024-09-17T01:03:17.977 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:17 smithi033 ceph-mon[23750]: pgmap v275: 97 pgs: 97 active+clean; 217 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.8 MiB/s wr, 561 op/s 2024-09-17T01:03:18.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:17 smithi100 ceph-mon[25838]: pgmap v275: 97 pgs: 97 active+clean; 217 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.8 MiB/s wr, 561 op/s 2024-09-17T01:03:18.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 32321 12.52 MB/sec warmup 77 sec latency 60.388 ms 2024-09-17T01:03:19.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 32553 12.40 MB/sec warmup 78 sec latency 74.969 ms 2024-09-17T01:03:20.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:19 smithi033 ceph-mon[23750]: pgmap v276: 97 pgs: 97 active+clean; 228 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 7.1 MiB/s wr, 705 op/s 2024-09-17T01:03:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:19 smithi100 ceph-mon[25838]: pgmap v276: 97 pgs: 97 active+clean; 228 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 7.3 KiB/s rd, 7.1 MiB/s wr, 705 op/s 2024-09-17T01:03:20.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 32941 12.31 MB/sec warmup 79 sec latency 60.338 ms 2024-09-17T01:03:21.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 33320 12.24 MB/sec warmup 80 sec latency 48.851 ms 2024-09-17T01:03:22.062 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:21 smithi033 ceph-mon[23750]: pgmap v277: 97 pgs: 97 active+clean; 230 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.0 MiB/s wr, 496 op/s 2024-09-17T01:03:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:21 smithi100 ceph-mon[25838]: pgmap v277: 97 pgs: 97 active+clean; 230 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.0 MiB/s wr, 496 op/s 2024-09-17T01:03:22.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 33663 12.26 MB/sec warmup 81 sec latency 54.666 ms 2024-09-17T01:03:23.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 34017 12.14 MB/sec warmup 82 sec latency 50.350 ms 2024-09-17T01:03:23.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:23 smithi033 ceph-mon[23750]: pgmap v278: 97 pgs: 97 active+clean; 229 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 444 op/s 2024-09-17T01:03:24.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:23 smithi100 ceph-mon[25838]: pgmap v278: 97 pgs: 97 active+clean; 229 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 444 op/s 2024-09-17T01:03:24.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 34409 12.11 MB/sec warmup 83 sec latency 40.199 ms 2024-09-17T01:03:25.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 34608 12.02 MB/sec warmup 84 sec latency 59.317 ms 2024-09-17T01:03:25.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:25 smithi033 ceph-mon[23750]: Upgrade: Updating mgr.smithi100.qrwkie 2024-09-17T01:03:25.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:25.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:03:25.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:03:25.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:25 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:03:25.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:25 smithi033 ceph-mon[23750]: Deploying daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T01:03:25.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:25 smithi100 ceph-mon[25838]: Upgrade: Updating mgr.smithi100.qrwkie 2024-09-17T01:03:25.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:25.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:03:25.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:03:25.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:25 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:03:25.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:25 smithi100 ceph-mon[25838]: Deploying daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T01:03:26.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 35374 12.24 MB/sec warmup 85 sec latency 39.623 ms 2024-09-17T01:03:26.420 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:26 smithi033 ceph-mon[23750]: pgmap v279: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 640 op/s 2024-09-17T01:03:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:26 smithi100 ceph-mon[25838]: pgmap v279: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.1 MiB/s wr, 640 op/s 2024-09-17T01:03:27.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 35887 12.32 MB/sec warmup 86 sec latency 40.029 ms 2024-09-17T01:03:27.445 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:27 smithi033 ceph-mon[23750]: pgmap v280: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 410 op/s 2024-09-17T01:03:27.689 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:27 smithi100 ceph-mon[25838]: pgmap v280: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 410 op/s 2024-09-17T01:03:28.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 36155 12.22 MB/sec warmup 87 sec latency 49.671 ms 2024-09-17T01:03:29.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 36669 12.18 MB/sec warmup 88 sec latency 29.615 ms 2024-09-17T01:03:29.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:29 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:29.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:29 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:03:29.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:29 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:03:29.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:29 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:03:29.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:29 smithi033 ceph-mon[23750]: pgmap v281: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-09-17T01:03:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:29 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:29 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:03:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:29 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:03:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:29 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:03:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:29 smithi100 ceph-mon[25838]: pgmap v281: 97 pgs: 97 active+clean; 236 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 516 op/s 2024-09-17T01:03:30.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 37344 12.24 MB/sec warmup 89 sec latency 22.689 ms 2024-09-17T01:03:31.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 38021 12.22 MB/sec warmup 90 sec latency 54.209 ms 2024-09-17T01:03:31.981 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:31 smithi033 ceph-mon[23750]: pgmap v282: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.3 MiB/s wr, 482 op/s 2024-09-17T01:03:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:31 smithi100 ceph-mon[25838]: pgmap v282: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.3 MiB/s wr, 482 op/s 2024-09-17T01:03:32.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 38485 12.31 MB/sec warmup 91 sec latency 32.121 ms 2024-09-17T01:03:33.129 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:03:33.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 38854 12.33 MB/sec warmup 92 sec latency 73.688 ms 2024-09-17T01:03:33.842 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:33 smithi033 ceph-mon[23750]: pgmap v283: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-17T01:03:33.842 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:33 smithi033 ceph-mon[23750]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:34.083 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:33 smithi100 ceph-mon[25838]: pgmap v283: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 479 op/s 2024-09-17T01:03:34.083 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:33 smithi100 ceph-mon[25838]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:34.176 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:03:34.176 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (6m) 94s ago 7m 22.8M - ba2b418f427c a6247209f546 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (7m) 94s ago 7m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (7m) 97s ago 7m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (6m) 94s ago 7m 52.7M - 8.3.5 dad864ee21e9 e7569df2559d 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (2m) 94s ago 2m 12.4M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (2m) 97s ago 2m 12.1M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:9283 running (9m) 94s ago 9m 460M - 17.2.0 e1d6a67b021e 4df4282576a9 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 starting - - - - 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (9m) 94s ago 9m 46.8M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (7m) 97s ago 7m 39.1M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (106s) 94s ago 105s 51.2M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (100s) 97s ago 99s 50.8M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (7m) 94s ago 7m 11.9M - 1dbe0e931976 65382f92c0e1 2024-09-17T01:03:34.177 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (7m) 97s ago 7m 17.6M - 1dbe0e931976 7fb37007ed80 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (6m) 94s ago 6m 42.7M 3217M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (5m) 94s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (5m) 94s ago 5m 40.3M 3217M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (5m) 94s ago 5m 40.8M 3217M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (4m) 97s ago 4m 39.9M 3983M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (4m) 97s ago 4m 41.9M 3983M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (3m) 97s ago 3m 40.0M 3983M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (3m) 97s ago 3m 41.9M 3983M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:03:34.178 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (6m) 94s ago 6m 63.8M - 514e6a882f6e 3afe8f20ef9f 2024-09-17T01:03:34.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 39520 12.44 MB/sec warmup 93 sec latency 22.425 ms 2024-09-17T01:03:34.618 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:03:34.618 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:03:34.618 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:03:34.618 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:34.618 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:03:34.619 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: Standby manager daemon smithi100.qrwkie restarted 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:03:34.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='client.24431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:34.858 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:34 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/106255044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:03:35.142 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:03:35.142 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:03:35.142 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:03:35.143 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:03:35.143 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "", 2024-09-17T01:03:35.143 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mgr daemons" 2024-09-17T01:03:35.143 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:03:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='client.24427 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie restarted 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='client.24431 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:35.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:34 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/106255044' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:03:35.254 INFO:teuthology.orchestra.run.smithi033.stdout: 5 39756 12.33 MB/sec warmup 94 sec latency 47.183 ms 2024-09-17T01:03:35.561 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:03:36.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:35 smithi033 ceph-mon[23750]: mgrmap e22: smithi033.illmog(active, since 7m), standbys: smithi100.qrwkie 2024-09-17T01:03:36.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:35 smithi033 ceph-mon[23750]: pgmap v284: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 777 op/s 2024-09-17T01:03:36.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:35 smithi033 ceph-mon[23750]: from='client.24435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:36.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:35 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/456845296' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:03:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:35 smithi100 ceph-mon[25838]: mgrmap e22: smithi033.illmog(active, since 7m), standbys: smithi100.qrwkie 2024-09-17T01:03:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:35 smithi100 ceph-mon[25838]: pgmap v284: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.9 MiB/s wr, 777 op/s 2024-09-17T01:03:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:35 smithi100 ceph-mon[25838]: from='client.24435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:03:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:35 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/456845296' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:03:36.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 40214 12.28 MB/sec warmup 95 sec latency 58.056 ms 2024-09-17T01:03:37.256 INFO:teuthology.orchestra.run.smithi033.stdout: 5 40468 12.24 MB/sec warmup 96 sec latency 146.206 ms 2024-09-17T01:03:38.218 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:37 smithi033 ceph-mon[23750]: pgmap v285: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 578 op/s 2024-09-17T01:03:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:37 smithi100 ceph-mon[25838]: pgmap v285: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.6 MiB/s wr, 578 op/s 2024-09-17T01:03:38.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 40669 12.15 MB/sec warmup 97 sec latency 60.627 ms 2024-09-17T01:03:39.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 40964 12.10 MB/sec warmup 98 sec latency 109.720 ms 2024-09-17T01:03:40.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:39 smithi033 ceph-mon[23750]: pgmap v286: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.8 MiB/s wr, 762 op/s 2024-09-17T01:03:40.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:39 smithi100 ceph-mon[25838]: pgmap v286: 97 pgs: 97 active+clean; 255 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.8 MiB/s wr, 762 op/s 2024-09-17T01:03:40.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 41229 12.03 MB/sec warmup 99 sec latency 120.582 ms 2024-09-17T01:03:41.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 41824 12.07 MB/sec warmup 100 sec latency 50.409 ms 2024-09-17T01:03:42.085 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:41 smithi033 ceph-mon[23750]: pgmap v287: 97 pgs: 97 active+clean; 251 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 669 op/s 2024-09-17T01:03:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:41 smithi100 ceph-mon[25838]: pgmap v287: 97 pgs: 97 active+clean; 251 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 669 op/s 2024-09-17T01:03:42.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 42629 12.27 MB/sec warmup 101 sec latency 43.385 ms 2024-09-17T01:03:43.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 43029 12.27 MB/sec warmup 102 sec latency 61.010 ms 2024-09-17T01:03:43.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:43 smithi033 ceph-mon[23750]: pgmap v288: 97 pgs: 97 active+clean; 249 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 554 op/s 2024-09-17T01:03:43.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:43 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:43 smithi100 ceph-mon[25838]: pgmap v288: 97 pgs: 97 active+clean; 249 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.4 MiB/s wr, 554 op/s 2024-09-17T01:03:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:43 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:44.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 43656 12.25 MB/sec warmup 103 sec latency 32.841 ms 2024-09-17T01:03:45.169 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:45 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:45.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 44009 12.22 MB/sec warmup 104 sec latency 54.850 ms 2024-09-17T01:03:45.427 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:45 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:46.196 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:46 smithi033 ceph-mon[23750]: pgmap v289: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 793 op/s 2024-09-17T01:03:46.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 44550 12.18 MB/sec warmup 105 sec latency 42.454 ms 2024-09-17T01:03:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:46 smithi100 ceph-mon[25838]: pgmap v289: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 8.2 MiB/s wr, 793 op/s 2024-09-17T01:03:47.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 44987 12.18 MB/sec warmup 106 sec latency 54.240 ms 2024-09-17T01:03:48.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 45245 12.13 MB/sec warmup 107 sec latency 114.104 ms 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: pgmap v290: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr fail", "who": "smithi033.illmog"}]: dispatch 2024-09-17T01:03:48.276 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:48 smithi033 ceph-mon[23750]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T01:03:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: pgmap v290: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.5 MiB/s wr, 503 op/s 2024-09-17T01:03:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' 2024-09-17T01:03:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:03:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr fail", "who": "smithi033.illmog"}]: dispatch 2024-09-17T01:03:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:48 smithi100 ceph-mon[25838]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T01:03:49.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 45763 12.21 MB/sec warmup 108 sec latency 61.761 ms 2024-09-17T01:03:49.287 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: Upgrade: Need to upgrade myself (mgr.smithi033.illmog) 2024-09-17T01:03:49.287 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: Failing over to other MGR 2024-09-17T01:03:49.287 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "mgr fail", "who": "smithi033.illmog"}]': finished 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: mgrmap e23: smithi100.qrwkie(active, starting, since 1.007s) 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:03:49.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: Manager daemon smithi100.qrwkie is now available 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:49.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:03:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: Upgrade: Need to upgrade myself (mgr.smithi033.illmog) 2024-09-17T01:03:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: Failing over to other MGR 2024-09-17T01:03:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:03:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.14202 172.21.15.33:0/1276839458' entity='mgr.smithi033.illmog' cmd='[{"prefix": "mgr fail", "who": "smithi033.illmog"}]': finished 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: mgrmap e23: smithi100.qrwkie(active, starting, since 1.007s) 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:03:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: Manager daemon smithi100.qrwkie is now available 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:49.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: Checking for cert/key for grafana.smithi033 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/trash_purge_schedule"}]: dispatch 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/trash_purge_schedule"}]: dispatch 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: mgrmap e24: smithi100.qrwkie(active, since 2s) 2024-09-17T01:03:50.116 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:50.117 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:50 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:50.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 46028 12.22 MB/sec warmup 109 sec latency 67.044 ms 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: Checking for cert/key for grafana.smithi033 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/trash_purge_schedule"}]: dispatch 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi100.qrwkie/trash_purge_schedule"}]: dispatch 2024-09-17T01:03:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: mgrmap e24: smithi100.qrwkie(active, since 2s) 2024-09-17T01:03:50.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:50.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:50 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:51.109 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:51 smithi033 ceph-mon[23750]: Deploying cephadm binary to smithi100 2024-09-17T01:03:51.109 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:51 smithi033 ceph-mon[23750]: pgmap v3: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:51.109 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:51 smithi033 ceph-mon[23750]: mgrmap e25: smithi100.qrwkie(active, since 3s) 2024-09-17T01:03:51.165 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:51 smithi100 ceph-mon[25838]: Deploying cephadm binary to smithi100 2024-09-17T01:03:51.165 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:51 smithi100 ceph-mon[25838]: pgmap v3: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:51.165 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:51 smithi100 ceph-mon[25838]: mgrmap e25: smithi100.qrwkie(active, since 3s) 2024-09-17T01:03:51.255 INFO:teuthology.orchestra.run.smithi033.stdout: 5 46279 12.19 MB/sec warmup 110 sec latency 126.980 ms 2024-09-17T01:03:52.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: Deploying cephadm binary to smithi033 2024-09-17T01:03:52.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: [17/Sep/2024:01:03:50] ENGINE Bus STARTING 2024-09-17T01:03:52.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: pgmap v4: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:52.122 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: [17/Sep/2024:01:03:50] ENGINE Serving on http://172.21.15.100:8765 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: [17/Sep/2024:01:03:50] ENGINE Serving on https://172.21.15.100:7150 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: [17/Sep/2024:01:03:50] ENGINE Bus STARTED 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:52 smithi033 ceph-mon[23750]: [17/Sep/2024:01:03:50] ENGINE Error in HTTPServer.serve 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Traceback (most recent call last): 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._run(expiration_interval) 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:03:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: s = self.context.wrap_socket( 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: return self.sslsocket_class._create( 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self.do_handshake() 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._sslobj.do_handshake() 2024-09-17T01:03:52.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:03:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: Deploying cephadm binary to smithi033 2024-09-17T01:03:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: [17/Sep/2024:01:03:50] ENGINE Bus STARTING 2024-09-17T01:03:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: pgmap v4: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: [17/Sep/2024:01:03:50] ENGINE Serving on http://172.21.15.100:8765 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: [17/Sep/2024:01:03:50] ENGINE Serving on https://172.21.15.100:7150 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: [17/Sep/2024:01:03:50] ENGINE Bus STARTED 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:52 smithi100 ceph-mon[25838]: [17/Sep/2024:01:03:50] ENGINE Error in HTTPServer.serve 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Traceback (most recent call last): 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._run(expiration_interval) 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:03:52.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: s = self.context.wrap_socket( 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: return self.sslsocket_class._create( 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self.do_handshake() 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._sslobj.do_handshake() 2024-09-17T01:03:52.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:03:52.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 46701 12.15 MB/sec warmup 111 sec latency 41.546 ms 2024-09-17T01:03:53.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 47132 12.11 MB/sec warmup 112 sec latency 54.836 ms 2024-09-17T01:03:53.891 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:53 smithi033 ceph-mon[23750]: pgmap v5: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:53.891 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:53 smithi033 ceph-mon[23750]: mgrmap e26: smithi100.qrwkie(active, since 5s) 2024-09-17T01:03:53.943 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:53 smithi100 ceph-mon[25838]: pgmap v5: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:53.944 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:53 smithi100 ceph-mon[25838]: mgrmap e26: smithi100.qrwkie(active, since 5s) 2024-09-17T01:03:54.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 47507 12.06 MB/sec warmup 113 sec latency 43.213 ms 2024-09-17T01:03:55.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 48124 12.08 MB/sec warmup 114 sec latency 32.445 ms 2024-09-17T01:03:55.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:55 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:55 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:55 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:55 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.455 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:55 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.456 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:55 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.456 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:55 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:55.456 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:55 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:56.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 48667 12.07 MB/sec warmup 115 sec latency 32.233 ms 2024-09-17T01:03:56.343 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:56 smithi100 ceph-mon[25838]: pgmap v6: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:56.446 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:56 smithi033 ceph-mon[23750]: pgmap v6: 97 pgs: 97 active+clean; 275 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-17T01:03:57.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 49076 12.12 MB/sec warmup 116 sec latency 32.603 ms 2024-09-17T01:03:58.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 49527 12.17 MB/sec warmup 117 sec latency 51.882 ms 2024-09-17T01:03:58.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:58 smithi033 ceph-mon[23750]: pgmap v7: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 MiB/s wr, 432 op/s 2024-09-17T01:03:58.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:58 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/crt"}]: dispatch 2024-09-17T01:03:58.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:58 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:03:58.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:58 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/key"}]: dispatch 2024-09-17T01:03:58.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:58 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:03:58.536 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:58 smithi033 ceph-mon[23750]: Standby manager daemon smithi033.illmog started 2024-09-17T01:03:58.660 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:58 smithi100 ceph-mon[25838]: pgmap v7: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 4.0 MiB/s wr, 432 op/s 2024-09-17T01:03:58.660 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:58 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/crt"}]: dispatch 2024-09-17T01:03:58.660 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:58 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:03:58.660 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:58 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/key"}]: dispatch 2024-09-17T01:03:58.660 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:58 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/798975843' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:03:58.660 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:58 smithi100 ceph-mon[25838]: Standby manager daemon smithi033.illmog started 2024-09-17T01:03:59.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 49818 12.14 MB/sec warmup 118 sec latency 104.708 ms 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: mgrmap e27: smithi100.qrwkie(active, since 10s), standbys: smithi033.illmog 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.330 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.331 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.331 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.331 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.331 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.331 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:03:59 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:59.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: mgrmap e27: smithi100.qrwkie(active, since 10s), standbys: smithi033.illmog 2024-09-17T01:03:59.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T01:03:59.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:59.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:03:59.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:03:59.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:03:59 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:00.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 50105 12.12 MB/sec warmup 119 sec latency 78.535 ms 2024-09-17T01:04:00.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:00 smithi033 ceph-mon[23750]: pgmap v8: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 3.1 MiB/s wr, 335 op/s 2024-09-17T01:04:00.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:00 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi100 to 2703M 2024-09-17T01:04:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:00 smithi100 ceph-mon[25838]: pgmap v8: 97 pgs: 97 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 3.1 MiB/s wr, 335 op/s 2024-09-17T01:04:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:00 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi100 to 2703M 2024-09-17T01:04:01.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:01 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:01.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:01 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:01 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:01.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:01 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 50987 5.63 MB/sec execute 1 sec latency 41.529 ms 2024-09-17T01:04:02.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: pgmap v9: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-17T01:04:02.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: Adjusting osd_memory_target on smithi033 to 1937M 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T01:04:02.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T01:04:02.391 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.391 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: pgmap v9: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 5.6 MiB/s wr, 544 op/s 2024-09-17T01:04:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: Adjusting osd_memory_target on smithi033 to 1937M 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:04:02.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T01:04:02.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T01:04:02.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:02.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 51401 9.88 MB/sec execute 2 sec latency 40.220 ms 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.410 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.410 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.410 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.410 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.410 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:03 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:04:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:04:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:04:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:03 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:04.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 52082 10.70 MB/sec execute 3 sec latency 39.533 ms 2024-09-17T01:04:04.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:04 smithi033 ceph-mon[23750]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:04:04.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:04 smithi033 ceph-mon[23750]: pgmap v10: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-09-17T01:04:04.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:04 smithi033 ceph-mon[23750]: Reconfiguring alertmanager.smithi033 (dependencies changed)... 2024-09-17T01:04:04.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:04 smithi033 ceph-mon[23750]: Deploying daemon alertmanager.smithi033 on smithi033 2024-09-17T01:04:04.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:04 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:04 smithi100 ceph-mon[25838]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:04:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:04 smithi100 ceph-mon[25838]: pgmap v10: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-09-17T01:04:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:04 smithi100 ceph-mon[25838]: Reconfiguring alertmanager.smithi033 (dependencies changed)... 2024-09-17T01:04:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:04 smithi100 ceph-mon[25838]: Deploying daemon alertmanager.smithi033 on smithi033 2024-09-17T01:04:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:04 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:05.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 52439 10.67 MB/sec execute 4 sec latency 43.409 ms 2024-09-17T01:04:06.119 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:04:06.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 52928 12.82 MB/sec execute 5 sec latency 44.021 ms 2024-09-17T01:04:06.452 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:06 smithi033 ceph-mon[23750]: pgmap v11: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-09-17T01:04:06.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:06 smithi100 ceph-mon[25838]: pgmap v11: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 5.1 MiB/s wr, 495 op/s 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (7m) 6s ago 8m 26.5M - 0.23.0 ba2b418f427c a6247209f546 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (8m) 6s ago 8m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (7m) 12s ago 7m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (6m) 6s ago 8m 52.9M - 8.3.5 dad864ee21e9 e7569df2559d 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (2m) 6s ago 2m 33.0M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:04:07.050 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (2m) 12s ago 2m 12.9M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:9283 running (9m) 6s ago 9m 184M - 17.2.0 e1d6a67b021e 4df4282576a9 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (39s) 12s ago 7m 478M - 19.1.1-330-gc2ddf972 ebb0615149de 722bf9301565 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (9m) 6s ago 9m 50.6M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (7m) 12s ago 7m 42.3M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (2m) 6s ago 2m 235M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (2m) 12s ago 2m 76.0M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (8m) 6s ago 8m 20.0M - 1.3.1 1dbe0e931976 65382f92c0e1 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (7m) 12s ago 7m 17.1M - 1.3.1 1dbe0e931976 7fb37007ed80 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (7m) 6s ago 7m 245M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (6m) 6s ago 6m 184M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (6m) 6s ago 6m 134M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (5m) 6s ago 5m 191M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (5m) 12s ago 5m 205M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:04:07.051 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (4m) 12s ago 4m 194M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:04:07.052 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (4m) 12s ago 4m 198M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:04:07.052 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (4m) 12s ago 4m 288M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:04:07.052 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (7m) 6s ago 7m 66.0M - 2.33.4 514e6a882f6e 3afe8f20ef9f 2024-09-17T01:04:07.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 53177 11.96 MB/sec execute 6 sec latency 119.319 ms 2024-09-17T01:04:07.458 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:07 smithi033 ceph-mon[23750]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:07.694 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:04:07.695 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:04:07.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:07 smithi100 ceph-mon[25838]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.188 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:04:08.188 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:04:08.188 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:04:08.188 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:04:08.188 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:04:08.188 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T01:04:08.189 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-17T01:04:08.189 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:04:08.189 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:04:08.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 53577 12.54 MB/sec execute 7 sec latency 45.765 ms 2024-09-17T01:04:08.519 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:08 smithi033 ceph-mon[23750]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.519 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:08 smithi033 ceph-mon[23750]: pgmap v12: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 7.6 MiB/s wr, 774 op/s 2024-09-17T01:04:08.519 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:08 smithi033 ceph-mon[23750]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.519 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:08 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3703702117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:04:08.520 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:08 smithi033 ceph-mon[23750]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:08 smithi100 ceph-mon[25838]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:08 smithi100 ceph-mon[25838]: pgmap v12: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 7.6 MiB/s wr, 774 op/s 2024-09-17T01:04:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:08 smithi100 ceph-mon[25838]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:08 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3703702117' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:04:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:08 smithi100 ceph-mon[25838]: from='client.14710 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:08.750 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:04:09.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 53729 11.40 MB/sec execute 8 sec latency 59.977 ms 2024-09-17T01:04:09.565 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:09 smithi033 ceph-mon[23750]: pgmap v13: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 525 op/s 2024-09-17T01:04:09.565 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:09 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/2017718596' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:04:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:09 smithi100 ceph-mon[25838]: pgmap v13: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 525 op/s 2024-09-17T01:04:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:09 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/2017718596' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:04:10.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 53922 10.52 MB/sec execute 9 sec latency 71.738 ms 2024-09-17T01:04:11.257 INFO:teuthology.orchestra.run.smithi033.stdout: 5 54220 9.66 MB/sec execute 10 sec latency 63.752 ms 2024-09-17T01:04:11.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:11 smithi033 ceph-mon[23750]: pgmap v14: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 714 op/s 2024-09-17T01:04:11.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:11 smithi100 ceph-mon[25838]: pgmap v14: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 714 op/s 2024-09-17T01:04:12.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 54467 9.21 MB/sec execute 11 sec latency 54.053 ms 2024-09-17T01:04:13.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 54847 9.57 MB/sec execute 12 sec latency 41.136 ms 2024-09-17T01:04:13.940 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:13 smithi033 ceph-mon[23750]: pgmap v15: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 468 op/s 2024-09-17T01:04:14.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:13 smithi100 ceph-mon[25838]: pgmap v15: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 468 op/s 2024-09-17T01:04:14.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 55353 9.66 MB/sec execute 13 sec latency 35.531 ms 2024-09-17T01:04:15.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 55848 9.58 MB/sec execute 14 sec latency 42.259 ms 2024-09-17T01:04:15.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:15 smithi033 ceph-mon[23750]: pgmap v16: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 468 op/s 2024-09-17T01:04:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:15 smithi100 ceph-mon[25838]: pgmap v16: 97 pgs: 97 active+clean; 305 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.8 MiB/s wr, 468 op/s 2024-09-17T01:04:16.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 56087 9.56 MB/sec execute 15 sec latency 58.419 ms 2024-09-17T01:04:17.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 56436 9.86 MB/sec execute 16 sec latency 52.517 ms 2024-09-17T01:04:17.991 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:17 smithi033 ceph-mon[23750]: pgmap v17: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 678 op/s 2024-09-17T01:04:18.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:17 smithi100 ceph-mon[25838]: pgmap v17: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 6.5 MiB/s wr, 678 op/s 2024-09-17T01:04:18.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 56741 9.71 MB/sec execute 17 sec latency 70.692 ms 2024-09-17T01:04:19.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 57243 10.48 MB/sec execute 18 sec latency 51.991 ms 2024-09-17T01:04:19.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:19 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:04:19.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:19 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:19.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:19.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:19 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:04:19.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:19 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:20.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 57483 10.03 MB/sec execute 19 sec latency 52.254 ms 2024-09-17T01:04:20.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:20 smithi033 ceph-mon[23750]: Reconfiguring grafana.smithi033 (dependencies changed)... 2024-09-17T01:04:20.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:20 smithi033 ceph-mon[23750]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T01:04:20.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:20 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:04:20.524 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:20 smithi033 ceph-mon[23750]: Reconfiguring daemon grafana.smithi033 on smithi033 2024-09-17T01:04:20.525 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:20 smithi033 ceph-mon[23750]: pgmap v18: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-17T01:04:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:20 smithi100 ceph-mon[25838]: Reconfiguring grafana.smithi033 (dependencies changed)... 2024-09-17T01:04:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:20 smithi100 ceph-mon[25838]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T01:04:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:20 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T01:04:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:20 smithi100 ceph-mon[25838]: Reconfiguring daemon grafana.smithi033 on smithi033 2024-09-17T01:04:20.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:20 smithi100 ceph-mon[25838]: pgmap v18: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 399 op/s 2024-09-17T01:04:21.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 57839 9.67 MB/sec execute 20 sec latency 61.084 ms 2024-09-17T01:04:22.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 57995 9.30 MB/sec execute 21 sec latency 94.034 ms 2024-09-17T01:04:22.565 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:22 smithi033 ceph-mon[23750]: pgmap v19: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.8 MiB/s wr, 616 op/s 2024-09-17T01:04:22.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:22 smithi100 ceph-mon[25838]: pgmap v19: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.8 MiB/s wr, 616 op/s 2024-09-17T01:04:23.263 INFO:teuthology.orchestra.run.smithi033.stdout: 5 58229 9.28 MB/sec execute 22 sec latency 43.521 ms 2024-09-17T01:04:24.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 58681 9.53 MB/sec execute 23 sec latency 43.025 ms 2024-09-17T01:04:24.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:24 smithi033 ceph-mon[23750]: pgmap v20: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-17T01:04:24.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:24 smithi100 ceph-mon[25838]: pgmap v20: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-17T01:04:25.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 59221 9.41 MB/sec execute 24 sec latency 52.651 ms 2024-09-17T01:04:26.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 59478 9.30 MB/sec execute 25 sec latency 61.414 ms 2024-09-17T01:04:26.380 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:26 smithi033 ceph-mon[23750]: pgmap v21: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-17T01:04:26.380 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:26.380 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:26.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:26 smithi100 ceph-mon[25838]: pgmap v21: 97 pgs: 97 active+clean; 317 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 426 op/s 2024-09-17T01:04:26.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:26.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:27.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 59967 9.73 MB/sec execute 26 sec latency 43.362 ms 2024-09-17T01:04:27.388 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:27 smithi033 ceph-mon[23750]: Reconfiguring node-exporter.smithi033 (dependencies changed)... 2024-09-17T01:04:27.388 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:27 smithi033 ceph-mon[23750]: Deploying daemon node-exporter.smithi033 on smithi033 2024-09-17T01:04:27.388 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:27 smithi033 ceph-mon[23750]: pgmap v22: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 620 op/s 2024-09-17T01:04:27.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:27 smithi100 ceph-mon[25838]: Reconfiguring node-exporter.smithi033 (dependencies changed)... 2024-09-17T01:04:27.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:27 smithi100 ceph-mon[25838]: Deploying daemon node-exporter.smithi033 on smithi033 2024-09-17T01:04:27.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:27 smithi100 ceph-mon[25838]: pgmap v22: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 620 op/s 2024-09-17T01:04:28.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 60433 9.96 MB/sec execute 27 sec latency 43.416 ms 2024-09-17T01:04:29.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 60778 10.14 MB/sec execute 28 sec latency 40.565 ms 2024-09-17T01:04:29.947 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:29 smithi033 ceph-mon[23750]: pgmap v23: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-17T01:04:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:29 smithi100 ceph-mon[25838]: pgmap v23: 97 pgs: 97 active+clean; 316 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 410 op/s 2024-09-17T01:04:30.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 61045 9.87 MB/sec execute 29 sec latency 41.957 ms 2024-09-17T01:04:31.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 61476 9.67 MB/sec execute 30 sec latency 59.097 ms 2024-09-17T01:04:31.963 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:31 smithi033 ceph-mon[23750]: pgmap v24: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 639 op/s 2024-09-17T01:04:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:31 smithi100 ceph-mon[25838]: pgmap v24: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.0 MiB/s wr, 639 op/s 2024-09-17T01:04:32.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 61784 9.68 MB/sec execute 31 sec latency 48.274 ms 2024-09-17T01:04:33.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 62168 9.83 MB/sec execute 32 sec latency 64.015 ms 2024-09-17T01:04:33.485 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:33 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:33.485 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:33 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:33.485 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:33 smithi033 ceph-mon[23750]: Reconfiguring prometheus.smithi033 (dependencies changed)... 2024-09-17T01:04:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:33 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:33 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:33 smithi100 ceph-mon[25838]: Reconfiguring prometheus.smithi033 (dependencies changed)... 2024-09-17T01:04:34.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 62649 9.72 MB/sec execute 33 sec latency 64.428 ms 2024-09-17T01:04:34.533 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:34 smithi033 ceph-mon[23750]: Deploying daemon prometheus.smithi033 on smithi033 2024-09-17T01:04:34.533 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:34 smithi033 ceph-mon[23750]: pgmap v25: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-17T01:04:34.533 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:34 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:34.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:34 smithi100 ceph-mon[25838]: Deploying daemon prometheus.smithi033 on smithi033 2024-09-17T01:04:34.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:34 smithi100 ceph-mon[25838]: pgmap v25: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.3 MiB/s wr, 423 op/s 2024-09-17T01:04:34.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:34 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:35.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 63003 9.64 MB/sec execute 34 sec latency 63.500 ms 2024-09-17T01:04:36.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 63329 9.67 MB/sec execute 35 sec latency 84.435 ms 2024-09-17T01:04:36.575 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:36 smithi033 ceph-mon[23750]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 434 op/s 2024-09-17T01:04:36.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:36 smithi100 ceph-mon[25838]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.5 MiB/s wr, 434 op/s 2024-09-17T01:04:37.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 63860 9.91 MB/sec execute 36 sec latency 33.070 ms 2024-09-17T01:04:38.264 INFO:teuthology.orchestra.run.smithi033.stdout: 5 64194 10.07 MB/sec execute 37 sec latency 103.573 ms 2024-09-17T01:04:38.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:38 smithi033 ceph-mon[23750]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 653 op/s 2024-09-17T01:04:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:38 smithi100 ceph-mon[25838]: pgmap v27: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.2 MiB/s wr, 653 op/s 2024-09-17T01:04:39.269 INFO:teuthology.orchestra.run.smithi033.stdout: 5 64461 10.01 MB/sec execute 38 sec latency 54.521 ms 2024-09-17T01:04:39.368 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:04:40.269 INFO:teuthology.orchestra.run.smithi033.stdout: 5 64713 9.81 MB/sec execute 39 sec latency 47.771 ms 2024-09-17T01:04:40.353 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:04:40.353 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 starting - - - - 2024-09-17T01:04:40.353 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (9m) 40s ago 9m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (8m) 46s ago 8m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 starting - - - - 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (3m) 40s ago 3m 33.0M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (3m) 46s ago 3m 12.9M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:9283 running (10m) 40s ago 10m 184M - 17.2.0 e1d6a67b021e 4df4282576a9 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (72s) 46s ago 8m 478M - 19.1.1-330-gc2ddf972 ebb0615149de 722bf9301565 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (10m) 40s ago 10m 50.6M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (8m) 46s ago 8m 42.3M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (2m) 40s ago 2m 235M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (2m) 46s ago 2m 76.0M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 starting - - - - 2024-09-17T01:04:40.354 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (8m) 46s ago 8m 17.1M - 1.3.1 1dbe0e931976 7fb37007ed80 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (7m) 40s ago 7m 245M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (7m) 40s ago 7m 184M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (6m) 40s ago 6m 134M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (6m) 40s ago 6m 191M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (5m) 46s ago 5m 205M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (5m) 46s ago 5m 194M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (5m) 46s ago 5m 198M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (4m) 46s ago 4m 288M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:04:40.355 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (8m) 40s ago 8m 66.0M - 2.33.4 514e6a882f6e 3afe8f20ef9f 2024-09-17T01:04:40.387 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:40 smithi033 ceph-mon[23750]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 460 op/s 2024-09-17T01:04:40.387 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:40 smithi033 ceph-mon[23750]: from='client.24479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:40 smithi100 ceph-mon[25838]: pgmap v28: 97 pgs: 97 active+clean; 329 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 460 op/s 2024-09-17T01:04:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:40 smithi100 ceph-mon[25838]: from='client.24479 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:40.864 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:04:40.864 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:04:40.864 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:04:40.864 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:40.864 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:04:40.864 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:04:40.865 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:04:40.866 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:04:40.866 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:04:40.866 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:04:41.269 INFO:teuthology.orchestra.run.smithi033.stdout: 5 64969 9.63 MB/sec execute 40 sec latency 83.633 ms 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:04:41.360 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:04:41.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:41 smithi033 ceph-mon[23750]: from='client.24483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:41.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:41 smithi033 ceph-mon[23750]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:41.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:41 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1864641217' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:04:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:41 smithi100 ceph-mon[25838]: from='client.24483 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:41 smithi100 ceph-mon[25838]: from='client.14724 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:41 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1864641217' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:04:42.087 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:04:42.275 INFO:teuthology.orchestra.run.smithi033.stdout: 5 65331 9.66 MB/sec execute 41 sec latency 61.703 ms 2024-09-17T01:04:42.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:42 smithi033 ceph-mon[23750]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.3 MiB/s wr, 745 op/s 2024-09-17T01:04:42.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:42 smithi033 ceph-mon[23750]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:42.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:42 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/349334522' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:04:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:42 smithi100 ceph-mon[25838]: pgmap v29: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.4 KiB/s rd, 7.3 MiB/s wr, 745 op/s 2024-09-17T01:04:42.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:42 smithi100 ceph-mon[25838]: from='client.14732 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:04:42.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:42 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/349334522' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:04:43.279 INFO:teuthology.orchestra.run.smithi033.stdout: 5 65817 9.80 MB/sec execute 42 sec latency 33.221 ms 2024-09-17T01:04:44.279 INFO:teuthology.orchestra.run.smithi033.stdout: 5 66377 9.73 MB/sec execute 43 sec latency 49.262 ms 2024-09-17T01:04:44.565 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:44 smithi033 ceph-mon[23750]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 515 op/s 2024-09-17T01:04:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:44 smithi100 ceph-mon[25838]: pgmap v30: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 515 op/s 2024-09-17T01:04:45.289 INFO:teuthology.orchestra.run.smithi033.stdout: 5 66634 9.66 MB/sec execute 44 sec latency 54.614 ms 2024-09-17T01:04:46.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 67096 9.90 MB/sec execute 45 sec latency 50.580 ms 2024-09-17T01:04:46.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:46 smithi033 ceph-mon[23750]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 533 op/s 2024-09-17T01:04:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:46 smithi100 ceph-mon[25838]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 533 op/s 2024-09-17T01:04:47.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 67445 9.88 MB/sec execute 46 sec latency 74.284 ms 2024-09-17T01:04:48.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 67871 10.09 MB/sec execute 47 sec latency 53.930 ms 2024-09-17T01:04:48.469 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:48 smithi033 ceph-mon[23750]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 731 op/s 2024-09-17T01:04:48.731 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:48 smithi100 ceph-mon[25838]: pgmap v32: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.4 MiB/s wr, 731 op/s 2024-09-17T01:04:49.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 68150 9.95 MB/sec execute 48 sec latency 48.515 ms 2024-09-17T01:04:49.499 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:49 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:49.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:49 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:04:50.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 68546 9.81 MB/sec execute 49 sec latency 42.760 ms 2024-09-17T01:04:50.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:50 smithi033 ceph-mon[23750]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 512 op/s 2024-09-17T01:04:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:50 smithi100 ceph-mon[25838]: pgmap v33: 97 pgs: 97 active+clean; 334 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 512 op/s 2024-09-17T01:04:51.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 68834 9.81 MB/sec execute 50 sec latency 84.060 ms 2024-09-17T01:04:52.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 69309 9.93 MB/sec execute 51 sec latency 40.290 ms 2024-09-17T01:04:52.563 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:52 smithi033 ceph-mon[23750]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.2 MiB/s wr, 725 op/s 2024-09-17T01:04:52.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:52 smithi100 ceph-mon[25838]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.2 MiB/s wr, 725 op/s 2024-09-17T01:04:53.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 69903 9.88 MB/sec execute 52 sec latency 58.937 ms 2024-09-17T01:04:54.069 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:53 smithi033 ceph-mon[23750]: pgmap v35: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-17T01:04:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:53 smithi100 ceph-mon[25838]: pgmap v35: 97 pgs: 97 active+clean; 341 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 440 op/s 2024-09-17T01:04:54.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 70439 9.99 MB/sec execute 53 sec latency 64.247 ms 2024-09-17T01:04:55.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 70949 10.17 MB/sec execute 54 sec latency 88.061 ms 2024-09-17T01:04:56.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 71418 10.37 MB/sec execute 55 sec latency 31.875 ms 2024-09-17T01:04:56.333 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:56 smithi033 ceph-mon[23750]: pgmap v36: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 560 op/s 2024-09-17T01:04:56.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:56 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:56.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:56 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:56.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:56 smithi033 ceph-mon[23750]: Reconfiguring node-exporter.smithi100 (dependencies changed)... 2024-09-17T01:04:56.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:56 smithi033 ceph-mon[23750]: Deploying daemon node-exporter.smithi100 on smithi100 2024-09-17T01:04:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:56 smithi100 ceph-mon[25838]: pgmap v36: 97 pgs: 97 active+clean; 347 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.5 MiB/s wr, 560 op/s 2024-09-17T01:04:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:56 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:56 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:04:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:56 smithi100 ceph-mon[25838]: Reconfiguring node-exporter.smithi100 (dependencies changed)... 2024-09-17T01:04:56.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:56 smithi100 ceph-mon[25838]: Deploying daemon node-exporter.smithi100 on smithi100 2024-09-17T01:04:57.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 71688 10.23 MB/sec execute 56 sec latency 41.082 ms 2024-09-17T01:04:58.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 71967 10.08 MB/sec execute 57 sec latency 72.005 ms 2024-09-17T01:04:58.367 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:04:58 smithi033 ceph-mon[23750]: pgmap v37: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 712 op/s 2024-09-17T01:04:58.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:04:58 smithi100 ceph-mon[25838]: pgmap v37: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 712 op/s 2024-09-17T01:04:59.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 72165 9.96 MB/sec execute 58 sec latency 111.283 ms 2024-09-17T01:05:00.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 72357 9.90 MB/sec execute 59 sec latency 71.350 ms 2024-09-17T01:05:00.532 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:00 smithi033 ceph-mon[23750]: pgmap v38: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 503 op/s 2024-09-17T01:05:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:00 smithi100 ceph-mon[25838]: pgmap v38: 97 pgs: 97 active+clean; 338 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.4 MiB/s wr, 503 op/s 2024-09-17T01:05:01.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 72611 9.82 MB/sec execute 60 sec latency 42.195 ms 2024-09-17T01:05:02.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 73302 9.97 MB/sec execute 61 sec latency 39.373 ms 2024-09-17T01:05:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 695 op/s 2024-09-17T01:05:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi033.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi033.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:02 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:02.554 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: pgmap v39: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.3 MiB/s wr, 695 op/s 2024-09-17T01:05:02.554 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.554 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.554 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:05:02.554 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi033.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi033.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:02.555 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:02 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:03.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 73647 9.91 MB/sec execute 62 sec latency 74.216 ms 2024-09-17T01:05:03.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:03 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:05:03.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:03 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi033.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:05:03.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:03 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:03.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:03 smithi033 ceph-mon[23750]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi033.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:05:03.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:03 smithi033 ceph-mon[23750]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:03.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:03 smithi033 ceph-mon[23750]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:03 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:05:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:03 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi033.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T01:05:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:03 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:03 smithi100 ceph-mon[25838]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi033.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T01:05:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:03 smithi100 ceph-mon[25838]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:03.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:03 smithi100 ceph-mon[25838]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:04.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 73755 9.77 MB/sec execute 63 sec latency 53.262 ms 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 482 op/s 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:04.370 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:04 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: pgmap v40: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.8 MiB/s wr, 482 op/s 2024-09-17T01:05:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:05:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:05:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:05:04.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:04.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:04 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:05.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 74030 9.82 MB/sec execute 64 sec latency 61.584 ms 2024-09-17T01:05:05.637 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:05 smithi033 ceph-mon[23750]: Upgrade: Updating mgr.smithi033.illmog 2024-09-17T01:05:05.637 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:05 smithi033 ceph-mon[23750]: Deploying daemon mgr.smithi033.illmog on smithi033 2024-09-17T01:05:05.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:05 smithi100 ceph-mon[25838]: Upgrade: Updating mgr.smithi033.illmog 2024-09-17T01:05:05.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:05 smithi100 ceph-mon[25838]: Deploying daemon mgr.smithi033.illmog on smithi033 2024-09-17T01:05:06.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 74387 9.84 MB/sec execute 65 sec latency 49.055 ms 2024-09-17T01:05:06.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:06 smithi033 ceph-mon[23750]: pgmap v41: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 580 op/s 2024-09-17T01:05:06.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:06 smithi100 ceph-mon[25838]: pgmap v41: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.3 MiB/s wr, 580 op/s 2024-09-17T01:05:07.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 74763 9.94 MB/sec execute 66 sec latency 60.509 ms 2024-09-17T01:05:07.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:07 smithi033 ceph-mon[23750]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 557 op/s 2024-09-17T01:05:07.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:07 smithi100 ceph-mon[25838]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 5.6 MiB/s wr, 557 op/s 2024-09-17T01:05:08.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 75073 9.96 MB/sec execute 67 sec latency 125.066 ms 2024-09-17T01:05:09.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 75314 9.85 MB/sec execute 68 sec latency 44.206 ms 2024-09-17T01:05:10.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 75631 9.74 MB/sec execute 69 sec latency 76.327 ms 2024-09-17T01:05:10.479 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:10 smithi033 ceph-mon[23750]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-09-17T01:05:10.479 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:10 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.479 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:10 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.479 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:10 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:10.479 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:10 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.479 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:10 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:10 smithi100 ceph-mon[25838]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 387 op/s 2024-09-17T01:05:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:10 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:10 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:10 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:10.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:10 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:10.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:10 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:11.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 75913 9.71 MB/sec execute 70 sec latency 58.385 ms 2024-09-17T01:05:12.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 76155 9.66 MB/sec execute 71 sec latency 66.584 ms 2024-09-17T01:05:12.517 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:12 smithi033 ceph-mon[23750]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 565 op/s 2024-09-17T01:05:12.584 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:05:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:12 smithi100 ceph-mon[25838]: pgmap v44: 97 pgs: 97 active+clean; 339 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 5.9 MiB/s wr, 565 op/s 2024-09-17T01:05:13.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 76445 9.74 MB/sec execute 72 sec latency 58.626 ms 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 starting - - - - 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (9m) 73s ago 9m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (9m) 0s ago 9m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 starting - - - - 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (3m) 73s ago 3m 33.0M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:05:13.634 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (3m) 0s ago 3m 13.3M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 starting - - - - 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (105s) 0s ago 8m 502M - 19.1.1-330-gc2ddf972 ebb0615149de 722bf9301565 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (10m) 73s ago 10m 50.6M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (8m) 0s ago 8m 43.8M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (3m) 73s ago 3m 235M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (3m) 0s ago 3m 82.3M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 starting - - - - 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (12s) 0s ago 8m 7293k - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (8m) 73s ago 8m 245M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (7m) 73s ago 7m 184M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (7m) 73s ago 7m 134M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (6m) 73s ago 6m 191M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:05:13.635 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (6m) 0s ago 6m 309M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:05:13.636 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (6m) 0s ago 6m 318M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:05:13.636 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (5m) 0s ago 5m 262M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:05:13.636 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (5m) 0s ago 5m 394M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:05:13.636 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 starting - - - - 2024-09-17T01:05:14.296 INFO:teuthology.orchestra.run.smithi033.stdout: 5 76667 9.63 MB/sec execute 73 sec latency 83.791 ms 2024-09-17T01:05:14.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:14 smithi033 ceph-mon[23750]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:14.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:14 smithi033 ceph-mon[23750]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 372 op/s 2024-09-17T01:05:14.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:14 smithi033 ceph-mon[23750]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:14.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:14 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:14.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:14 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:14.398 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:05:14.398 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:14.399 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:05:14.400 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T01:05:14.400 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:05:14.400 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:05:14.400 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:05:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:14 smithi100 ceph-mon[25838]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:14 smithi100 ceph-mon[25838]: pgmap v45: 97 pgs: 97 active+clean; 339 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 372 op/s 2024-09-17T01:05:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:14 smithi100 ceph-mon[25838]: from='client.24493 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:14 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:14 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "", 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:05:14.850 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:05:15.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 77020 9.53 MB/sec execute 74 sec latency 62.033 ms 2024-09-17T01:05:15.522 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:05:15.568 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:15 smithi033 ceph-mon[23750]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:15.568 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:15 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3565114840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:05:15.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:15 smithi100 ceph-mon[25838]: from='client.14752 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:15.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:15 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3565114840' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:05:16.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 77342 9.49 MB/sec execute 75 sec latency 50.850 ms 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: pgmap v46: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 389 op/s 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/479001478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: Standby manager daemon smithi033.illmog restarted 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: Standby manager daemon smithi033.illmog started 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/crt"}]: dispatch 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/key"}]: dispatch 2024-09-17T01:05:16.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:16 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:05:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: pgmap v46: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 389 op/s 2024-09-17T01:05:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: from='client.14760 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/479001478' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:05:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: Standby manager daemon smithi033.illmog restarted 2024-09-17T01:05:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: Standby manager daemon smithi033.illmog started 2024-09-17T01:05:16.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/crt"}]: dispatch 2024-09-17T01:05:16.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:05:16.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi033.illmog/key"}]: dispatch 2024-09-17T01:05:16.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:16 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:05:17.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 77771 9.62 MB/sec execute 76 sec latency 58.665 ms 2024-09-17T01:05:17.405 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:17 smithi033 ceph-mon[23750]: mgrmap e28: smithi100.qrwkie(active, since 88s), standbys: smithi033.illmog 2024-09-17T01:05:17.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:17 smithi100 ceph-mon[25838]: mgrmap e28: smithi100.qrwkie(active, since 88s), standbys: smithi033.illmog 2024-09-17T01:05:18.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 78158 9.64 MB/sec execute 77 sec latency 60.361 ms 2024-09-17T01:05:18.430 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:18 smithi033 ceph-mon[23750]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 450 op/s 2024-09-17T01:05:18.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:18 smithi100 ceph-mon[25838]: pgmap v47: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 450 op/s 2024-09-17T01:05:19.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 78545 9.73 MB/sec execute 78 sec latency 39.216 ms 2024-09-17T01:05:19.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:19 smithi033 ceph-mon[23750]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 353 op/s 2024-09-17T01:05:19.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:19.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:19.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:19 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:19.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:19 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:20.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:19 smithi100 ceph-mon[25838]: pgmap v48: 97 pgs: 97 active+clean; 328 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.6 MiB/s wr, 353 op/s 2024-09-17T01:05:20.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:20.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:20.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:19 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:20.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:19 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:20.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 78792 9.68 MB/sec execute 79 sec latency 44.361 ms 2024-09-17T01:05:20.985 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:20 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:21.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:20 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:21.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 79007 9.59 MB/sec execute 80 sec latency 46.029 ms 2024-09-17T01:05:21.998 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:21 smithi033 ceph-mon[23750]: pgmap v49: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 568 op/s 2024-09-17T01:05:21.998 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:21 smithi033 ceph-mon[23750]: mgrmap e29: smithi100.qrwkie(active, since 93s), standbys: smithi033.illmog 2024-09-17T01:05:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:21 smithi100 ceph-mon[25838]: pgmap v49: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.1 MiB/s wr, 568 op/s 2024-09-17T01:05:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:21 smithi100 ceph-mon[25838]: mgrmap e29: smithi100.qrwkie(active, since 93s), standbys: smithi033.illmog 2024-09-17T01:05:22.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 79395 9.56 MB/sec execute 81 sec latency 73.934 ms 2024-09-17T01:05:23.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 79919 9.68 MB/sec execute 82 sec latency 30.027 ms 2024-09-17T01:05:24.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:23 smithi033 ceph-mon[23750]: pgmap v50: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 390 op/s 2024-09-17T01:05:24.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:23 smithi100 ceph-mon[25838]: pgmap v50: 97 pgs: 97 active+clean; 336 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 390 op/s 2024-09-17T01:05:24.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 80466 9.64 MB/sec execute 83 sec latency 51.780 ms 2024-09-17T01:05:25.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 80878 9.61 MB/sec execute 84 sec latency 38.717 ms 2024-09-17T01:05:26.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:25 smithi033 ceph-mon[23750]: pgmap v51: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-17T01:05:26.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:25 smithi100 ceph-mon[25838]: pgmap v51: 97 pgs: 97 active+clean; 335 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 413 op/s 2024-09-17T01:05:26.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 81087 9.57 MB/sec execute 85 sec latency 62.518 ms 2024-09-17T01:05:27.098 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.098 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.098 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:27.098 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:05:27.098 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: Failing over to other MGR 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr fail", "who": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr fail", "who": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:05:27.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:26 smithi033 ceph-mon[23750]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:05:27.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:27.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:05:27.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: Upgrade: Need to upgrade myself (mgr.smithi100.qrwkie) 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: Failing over to other MGR 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 172.21.15.100:0/2275429957' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr fail", "who": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "mgr fail", "who": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:05:27.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:26 smithi100 ceph-mon[25838]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:05:27.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 81921 9.86 MB/sec execute 86 sec latency 44.265 ms 2024-09-17T01:05:28.104 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd='[{"prefix": "mgr fail", "who": "smithi100.qrwkie"}]': finished 2024-09-17T01:05:28.104 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: mgrmap e30: smithi033.illmog(active, starting, since 0.841628s) 2024-09-17T01:05:28.104 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:05:28.104 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:05:28.104 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:05:28.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: Manager daemon smithi033.illmog is now available 2024-09-17T01:05:28.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:28.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:28.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:05:28.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:28 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.24421 ' entity='mgr.smithi100.qrwkie' cmd='[{"prefix": "mgr fail", "who": "smithi100.qrwkie"}]': finished 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: mgrmap e30: smithi033.illmog(active, starting, since 0.841628s) 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:05:28.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: Manager daemon smithi033.illmog is now available 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:05:28.127 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:28 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:05:28.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 82221 9.87 MB/sec execute 87 sec latency 51.703 ms 2024-09-17T01:05:29.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 82399 9.78 MB/sec execute 88 sec latency 59.566 ms 2024-09-17T01:05:29.371 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:29 smithi033 ceph-mon[23750]: mgrmap e31: smithi033.illmog(active, since 2s) 2024-09-17T01:05:29.371 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:29 smithi033 ceph-mon[23750]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:29.371 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:29 smithi033 ceph-mon[23750]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:29 smithi100 ceph-mon[25838]: mgrmap e31: smithi033.illmog(active, since 2s) 2024-09-17T01:05:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:29 smithi100 ceph-mon[25838]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:29 smithi100 ceph-mon[25838]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:30.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 82817 9.75 MB/sec execute 89 sec latency 42.904 ms 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:30 smithi033 ceph-mon[23750]: [17/Sep/2024:01:05:29] ENGINE Bus STARTING 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:30 smithi033 ceph-mon[23750]: [17/Sep/2024:01:05:29] ENGINE Serving on http://172.21.15.33:8765 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:30 smithi033 ceph-mon[23750]: [17/Sep/2024:01:05:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:30 smithi033 ceph-mon[23750]: [17/Sep/2024:01:05:30] ENGINE Bus STARTED 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:30 smithi033 ceph-mon[23750]: [17/Sep/2024:01:05:30] ENGINE Error in HTTPServer.serve 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Traceback (most recent call last): 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:05:30.389 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._run(expiration_interval) 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: s = self.context.wrap_socket( 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: return self.sslsocket_class._create( 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self.do_handshake() 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._sslobj.do_handshake() 2024-09-17T01:05:30.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:05:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:05:29] ENGINE Bus STARTING 2024-09-17T01:05:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:05:29] ENGINE Serving on http://172.21.15.33:8765 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:05:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:05:30] ENGINE Bus STARTED 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:05:30] ENGINE Error in HTTPServer.serve 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Traceback (most recent call last): 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:05:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:05:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._run(expiration_interval) 2024-09-17T01:05:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:05:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:05:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:05:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:05:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:05:30.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: s = self.context.wrap_socket( 2024-09-17T01:05:30.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:05:30.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: return self.sslsocket_class._create( 2024-09-17T01:05:30.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:05:30.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self.do_handshake() 2024-09-17T01:05:30.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:05:30.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._sslobj.do_handshake() 2024-09-17T01:05:30.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:05:31.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 83252 9.78 MB/sec execute 90 sec latency 43.329 ms 2024-09-17T01:05:31.366 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:31 smithi100 ceph-mon[25838]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:31.405 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:31 smithi033 ceph-mon[23750]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:32.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 83722 9.83 MB/sec execute 91 sec latency 34.519 ms 2024-09-17T01:05:33.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 84242 9.77 MB/sec execute 92 sec latency 25.610 ms 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: mgrmap e32: smithi033.illmog(active, since 6s) 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:05:33.426 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:05:33.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:33.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.427 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:33 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: mgrmap e32: smithi033.illmog(active, since 6s) 2024-09-17T01:05:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/3958564364' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:33.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:33 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:34.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 84545 9.73 MB/sec execute 93 sec latency 51.599 ms 2024-09-17T01:05:34.962 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:34 smithi033 ceph-mon[23750]: mgrmap e33: smithi033.illmog(active, since 7s), standbys: smithi100.qrwkie 2024-09-17T01:05:34.962 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:34 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:05:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:34 smithi100 ceph-mon[25838]: mgrmap e33: smithi033.illmog(active, since 7s), standbys: smithi100.qrwkie 2024-09-17T01:05:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:34 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:05:35.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 85037 9.86 MB/sec execute 94 sec latency 46.796 ms 2024-09-17T01:05:35.973 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:35 smithi033 ceph-mon[23750]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.8 MiB/s wr, 444 op/s 2024-09-17T01:05:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:35 smithi100 ceph-mon[25838]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 38 KiB/s rd, 3.8 MiB/s wr, 444 op/s 2024-09-17T01:05:36.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 85478 9.93 MB/sec execute 95 sec latency 42.386 ms 2024-09-17T01:05:37.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 85824 9.94 MB/sec execute 96 sec latency 51.127 ms 2024-09-17T01:05:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:38 smithi100 ceph-mon[25838]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 342 op/s 2024-09-17T01:05:38.286 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:38 smithi033 ceph-mon[23750]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 342 op/s 2024-09-17T01:05:38.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 85994 9.87 MB/sec execute 97 sec latency 40.867 ms 2024-09-17T01:05:39.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 86305 9.83 MB/sec execute 98 sec latency 51.735 ms 2024-09-17T01:05:40.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:40 smithi033 ceph-mon[23750]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 519 op/s 2024-09-17T01:05:40.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:40 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:40 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:40 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:40 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 86625 9.80 MB/sec execute 99 sec latency 51.302 ms 2024-09-17T01:05:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:40 smithi100 ceph-mon[25838]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 5.3 MiB/s wr, 519 op/s 2024-09-17T01:05:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:40 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:40 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:40 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:40 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:41.297 INFO:teuthology.orchestra.run.smithi033.stdout: 5 87021 9.83 MB/sec execute 100 sec latency 42.656 ms 2024-09-17T01:05:42.105 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:42 smithi033 ceph-mon[23750]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-09-17T01:05:42.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 87498 9.81 MB/sec execute 101 sec latency 43.972 ms 2024-09-17T01:05:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:42 smithi100 ceph-mon[25838]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-09-17T01:05:43.115 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:43 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:43.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 87954 9.78 MB/sec execute 102 sec latency 48.071 ms 2024-09-17T01:05:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:43 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:44.180 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:44 smithi033 ceph-mon[23750]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-09-17T01:05:44.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 88156 9.75 MB/sec execute 103 sec latency 60.870 ms 2024-09-17T01:05:44.433 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:44 smithi100 ceph-mon[25838]: pgmap v11: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.7 MiB/s wr, 465 op/s 2024-09-17T01:05:45.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 88520 9.82 MB/sec execute 104 sec latency 44.838 ms 2024-09-17T01:05:46.092 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:05:46.173 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:46 smithi033 ceph-mon[23750]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 6.8 MiB/s wr, 704 op/s 2024-09-17T01:05:46.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 88818 9.84 MB/sec execute 105 sec latency 46.289 ms 2024-09-17T01:05:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:46 smithi100 ceph-mon[25838]: pgmap v12: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 6.8 MiB/s wr, 704 op/s 2024-09-17T01:05:47.193 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:47 smithi033 ceph-mon[23750]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:47.193 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:47 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:47.193 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:47 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:47.193 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:47 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:47.193 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:47 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:05:47.211 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:05:47.211 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (90s) 8s ago 10m 17.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (10m) 8s ago 10m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (9m) 14s ago 9m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (82s) 8s ago 9m 44.9M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (4m) 8s ago 4m 35.4M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (4m) 14s ago 4m 13.6M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (38s) 8s ago 11m 476M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283 running (2m) 14s ago 9m 114M - 19.1.1-330-gc2ddf972 ebb0615149de 722bf9301565 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (11m) 8s ago 11m 56.0M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (9m) 14s ago 9m 46.2M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (3m) 8s ago 3m 265M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (3m) 14s ago 3m 82.5M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (75s) 8s ago 9m 7738k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:05:47.212 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (46s) 14s ago 9m 7424k - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (8m) 8s ago 8m 362M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (8m) 8s ago 8m 271M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (7m) 8s ago 7m 211M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (7m) 8s ago 7m 303M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (6m) 14s ago 6m 326M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (6m) 14s ago 6m 341M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (6m) 14s ago 6m 273M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (5m) 14s ago 5m 418M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:05:47.213 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (52s) 8s ago 9m 43.5M - 2.43.0 a07b618ecd1d b4c0c86dba56 2024-09-17T01:05:47.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 89149 9.84 MB/sec execute 106 sec latency 57.963 ms 2024-09-17T01:05:47.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:47 smithi100 ceph-mon[25838]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:47.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:47 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:47.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:47 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:47.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:47 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:47.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:47 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:05:47.935 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:05:47.936 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:05:48.265 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:48 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/3646497572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:05:48.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 89359 9.81 MB/sec execute 107 sec latency 77.186 ms 2024-09-17T01:05:48.400 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:05:48.400 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:05:48.401 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:05:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T01:05:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T01:05:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:05:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='client.14800 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: pgmap v13: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:48 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/3646497572' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:05:48.945 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:05:49.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 89521 9.75 MB/sec execute 108 sec latency 60.860 ms 2024-09-17T01:05:49.302 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:49 smithi033 ceph-mon[23750]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:49.302 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:49 smithi033 ceph-mon[23750]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:05:49.302 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:49 smithi033 ceph-mon[23750]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:05:49.302 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:49 smithi033 ceph-mon[23750]: Reconfiguring prometheus.smithi033 (dependencies changed)... 2024-09-17T01:05:49.302 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:49 smithi033 ceph-mon[23750]: from='client.? 172.21.15.33:0/1965809942' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:05:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:49 smithi100 ceph-mon[25838]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:49 smithi100 ceph-mon[25838]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:05:49.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:49 smithi100 ceph-mon[25838]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:05:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:49 smithi100 ceph-mon[25838]: Reconfiguring prometheus.smithi033 (dependencies changed)... 2024-09-17T01:05:49.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:49 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/1965809942' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:05:50.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 89682 9.67 MB/sec execute 109 sec latency 107.262 ms 2024-09-17T01:05:50.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:50 smithi033 ceph-mon[23750]: Reconfiguring daemon prometheus.smithi033 on smithi033 2024-09-17T01:05:50.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:50 smithi033 ceph-mon[23750]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:50.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:50 smithi033 ceph-mon[23750]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.7 MiB/s wr, 617 op/s 2024-09-17T01:05:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:50 smithi100 ceph-mon[25838]: Reconfiguring daemon prometheus.smithi033 on smithi033 2024-09-17T01:05:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:50 smithi100 ceph-mon[25838]: from='client.14812 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:05:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:50 smithi100 ceph-mon[25838]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.7 MiB/s wr, 617 op/s 2024-09-17T01:05:51.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 89951 9.64 MB/sec execute 110 sec latency 60.034 ms 2024-09-17T01:05:52.123 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:52 smithi033 ceph-mon[23750]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T01:05:52.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 90206 9.59 MB/sec execute 111 sec latency 52.091 ms 2024-09-17T01:05:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:52 smithi100 ceph-mon[25838]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T01:05:53.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 90452 9.60 MB/sec execute 112 sec latency 57.205 ms 2024-09-17T01:05:53.930 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:53 smithi033 ceph-mon[23750]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T01:05:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:53 smithi100 ceph-mon[25838]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T01:05:54.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 90742 9.58 MB/sec execute 113 sec latency 69.453 ms 2024-09-17T01:05:55.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 91071 9.54 MB/sec execute 114 sec latency 75.269 ms 2024-09-17T01:05:56.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:56 smithi033 ceph-mon[23750]: pgmap v17: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 556 op/s 2024-09-17T01:05:56.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:56 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:56.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:56 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:56.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:56 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:56.217 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:56 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:56.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 91343 9.48 MB/sec execute 115 sec latency 54.148 ms 2024-09-17T01:05:56.329 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:56 smithi100 ceph-mon[25838]: pgmap v17: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.5 KiB/s rd, 5.6 MiB/s wr, 556 op/s 2024-09-17T01:05:56.329 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:56 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:56.329 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:56 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:05:56.330 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:56 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:56.330 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:56 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:05:57.239 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:57 smithi033 ceph-mon[23750]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:57.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 91556 9.43 MB/sec execute 116 sec latency 63.678 ms 2024-09-17T01:05:57.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:57 smithi100 ceph-mon[25838]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T01:05:58.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:58 smithi033 ceph-mon[23750]: pgmap v18: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 318 op/s 2024-09-17T01:05:58.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:58 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:05:58.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:58 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:05:58.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:58 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:58.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:58 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:58.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 91771 9.44 MB/sec execute 117 sec latency 281.486 ms 2024-09-17T01:05:58.310 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:58 smithi100 ceph-mon[25838]: pgmap v18: 97 pgs: 97 active+clean; 333 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 318 op/s 2024-09-17T01:05:58.310 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:58 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:05:58.310 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:58 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:05:58.310 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:58 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:05:58.310 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:58 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:05:59.174 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:59 smithi100 ceph-mon[25838]: Upgrade: Updating mgr.smithi100.qrwkie 2024-09-17T01:05:59.174 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:05:59 smithi100 ceph-mon[25838]: Deploying daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T01:05:59.298 INFO:teuthology.orchestra.run.smithi033.stdout: 5 92106 9.48 MB/sec execute 118 sec latency 52.357 ms 2024-09-17T01:05:59.332 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:59 smithi033 ceph-mon[23750]: Upgrade: Updating mgr.smithi100.qrwkie 2024-09-17T01:05:59.332 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:05:59 smithi033 ceph-mon[23750]: Deploying daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T01:06:00.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 92725 9.60 MB/sec execute 119 sec latency 26.251 ms 2024-09-17T01:06:00.340 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:00 smithi033 ceph-mon[23750]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-09-17T01:06:00.464 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:00 smithi100 ceph-mon[25838]: pgmap v19: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 479 op/s 2024-09-17T01:06:01.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 92979 9.56 MB/sec execute 120 sec latency 62.156 ms 2024-09-17T01:06:02.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 93208 9.50 MB/sec execute 121 sec latency 60.330 ms 2024-09-17T01:06:02.357 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:02 smithi033 ceph-mon[23750]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.9 MiB/s wr, 315 op/s 2024-09-17T01:06:02.357 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:02 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:02.357 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:02 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:02.357 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:02 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:02.400 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:02 smithi100 ceph-mon[25838]: pgmap v20: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.9 MiB/s wr, 315 op/s 2024-09-17T01:06:02.400 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:02 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:02.400 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:02 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:02.400 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:02 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:03.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 93514 9.47 MB/sec execute 122 sec latency 46.121 ms 2024-09-17T01:06:03.393 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:03 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:03.393 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:03 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:03.393 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:03 smithi033 ceph-mon[23750]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.9 MiB/s wr, 315 op/s 2024-09-17T01:06:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:03 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:03 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:03.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:03 smithi100 ceph-mon[25838]: pgmap v21: 97 pgs: 97 active+clean; 331 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 2.9 MiB/s wr, 315 op/s 2024-09-17T01:06:04.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 93749 9.44 MB/sec execute 123 sec latency 89.469 ms 2024-09-17T01:06:05.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 94055 9.46 MB/sec execute 124 sec latency 55.035 ms 2024-09-17T01:06:06.151 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:06 smithi100 ceph-mon[25838]: pgmap v22: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 512 op/s 2024-09-17T01:06:06.198 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:06 smithi033 ceph-mon[23750]: pgmap v22: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 512 op/s 2024-09-17T01:06:06.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 94310 9.45 MB/sec execute 125 sec latency 74.981 ms 2024-09-17T01:06:07.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 94600 9.39 MB/sec execute 126 sec latency 73.998 ms 2024-09-17T01:06:07.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:07 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:07 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:07 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:07 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:07 smithi033 ceph-mon[23750]: pgmap v23: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-09-17T01:06:07.511 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:07 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.511 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:07 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.511 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:07 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.512 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:07 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:07.512 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:07 smithi100 ceph-mon[25838]: pgmap v23: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.7 MiB/s wr, 358 op/s 2024-09-17T01:06:08.270 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:08 smithi033 ceph-mon[23750]: Standby manager daemon smithi100.qrwkie restarted 2024-09-17T01:06:08.270 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:08 smithi033 ceph-mon[23750]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:06:08.270 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:08 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:06:08.270 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:08 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:06:08.270 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:08 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:06:08.270 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:08 smithi033 ceph-mon[23750]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:06:08.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 94957 9.34 MB/sec execute 127 sec latency 74.385 ms 2024-09-17T01:06:08.515 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:08 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie restarted 2024-09-17T01:06:08.515 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:08 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:06:08.515 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:08 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:06:08.515 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:08 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:06:08.515 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:08 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:06:08.515 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:08 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/1847653906' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:06:09.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 95177 9.33 MB/sec execute 128 sec latency 53.680 ms 2024-09-17T01:06:09.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:09 smithi100 ceph-mon[25838]: mgrmap e34: smithi033.illmog(active, since 42s), standbys: smithi100.qrwkie 2024-09-17T01:06:09.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:09 smithi100 ceph-mon[25838]: pgmap v24: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 537 op/s 2024-09-17T01:06:09.533 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:09 smithi033 ceph-mon[23750]: mgrmap e34: smithi033.illmog(active, since 42s), standbys: smithi100.qrwkie 2024-09-17T01:06:09.533 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:09 smithi033 ceph-mon[23750]: pgmap v24: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.1 MiB/s wr, 537 op/s 2024-09-17T01:06:10.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 95415 9.33 MB/sec execute 129 sec latency 50.035 ms 2024-09-17T01:06:11.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 95805 9.40 MB/sec execute 130 sec latency 81.903 ms 2024-09-17T01:06:11.586 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.586 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.586 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.586 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: Upgrade: Setting container_image for all mgr 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi033.illmog"}]: dispatch 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi033.illmog"}]': finished 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi100.qrwkie"}]: dispatch 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi100.qrwkie"}]': finished 2024-09-17T01:06:11.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:06:11.588 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:11 smithi033 ceph-mon[23750]: pgmap v25: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-17T01:06:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: Upgrade: Setting container_image for all mgr 2024-09-17T01:06:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi033.illmog"}]: dispatch 2024-09-17T01:06:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi033.illmog"}]': finished 2024-09-17T01:06:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi100.qrwkie"}]: dispatch 2024-09-17T01:06:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi100.qrwkie"}]': finished 2024-09-17T01:06:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:06:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:11 smithi100 ceph-mon[25838]: pgmap v25: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-17T01:06:12.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 96169 9.40 MB/sec execute 131 sec latency 66.378 ms 2024-09-17T01:06:12.855 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:12 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:13.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:12 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:13.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 96460 9.40 MB/sec execute 132 sec latency 47.963 ms 2024-09-17T01:06:13.895 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:13 smithi033 ceph-mon[23750]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-17T01:06:13.895 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:13 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:13.895 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:13 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:06:13.895 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:13 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:06:13.895 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:13 smithi033 ceph-mon[23750]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:14.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:13 smithi100 ceph-mon[25838]: pgmap v26: 97 pgs: 97 active+clean; 329 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 376 op/s 2024-09-17T01:06:14.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:13 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' 2024-09-17T01:06:14.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:13 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:06:14.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:13 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:06:14.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:13 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:14.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 96613 9.35 MB/sec execute 133 sec latency 57.222 ms 2024-09-17T01:06:14.919 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:14 smithi033 ceph-mon[23750]: Upgrade: Updating mon.smithi033 2024-09-17T01:06:14.920 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:14 smithi033 ceph-mon[23750]: Deploying daemon mon.smithi033 on smithi033 2024-09-17T01:06:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:14 smithi100 ceph-mon[25838]: Upgrade: Updating mon.smithi033 2024-09-17T01:06:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:14 smithi100 ceph-mon[25838]: Deploying daemon mon.smithi033 on smithi033 2024-09-17T01:06:15.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 96782 9.29 MB/sec execute 134 sec latency 76.395 ms 2024-09-17T01:06:15.939 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:15 smithi033 ceph-mon[23750]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.8 MiB/s wr, 553 op/s 2024-09-17T01:06:16.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:16 smithi033 systemd[1]: Stopping Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:06:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:15 smithi100 ceph-mon[25838]: pgmap v27: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.8 MiB/s wr, 553 op/s 2024-09-17T01:06:16.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 96995 9.26 MB/sec execute 135 sec latency 75.635 ms 2024-09-17T01:06:16.448 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:16 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[23744]: 2024-09-17T01:06:16.348+0000 7f7f3ae70700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi033 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:06:16.448 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:16 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[23744]: 2024-09-17T01:06:16.348+0000 7f7f3ae70700 -1 mon.smithi033@0(leader) e2 *** Got Signal Terminated *** 2024-09-17T01:06:16.448 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:16 smithi033 podman[65264]: 2024-09-17 01:06:16.410207924 +0000 UTC m=+0.178231487 container died 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , RELEASE=HEAD, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, release=754, distribution-scope=public, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.buildah.version=1.19.8, io.openshift.expose-services=) 2024-09-17T01:06:17.214 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:17 smithi033 podman[65264]: 2024-09-17 01:06:17.136977758 +0000 UTC m=+0.905001313 container cleanup 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.buildah.version=1.19.8) 2024-09-17T01:06:17.214 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:17 smithi033 bash[65264]: ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033 2024-09-17T01:06:17.299 INFO:teuthology.orchestra.run.smithi033.stdout: 5 97207 9.22 MB/sec execute 136 sec latency 49.452 ms 2024-09-17T01:06:17.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:17 smithi033 podman[65277]: 2024-09-17 01:06:17.431995471 +0000 UTC m=+1.020345925 container remove 48394a6f2f5fc940c90ba7dcfba026ae4f17ab8029639221264ad56c7c2abfce (image=quay.io/ceph/ceph:v17.2.0, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_BRANCH=HEAD, distribution-scope=public, architecture=x86_64, maintainer=Guillaume Abrioux , release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream) 2024-09-17T01:06:18.003 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:17 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service: Deactivated successfully. 2024-09-17T01:06:18.003 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:17 smithi033 systemd[1]: Stopped Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:06:18.003 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:17 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service: Consumed 16.307s CPU time. 2024-09-17T01:06:18.264 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 systemd[1]: Starting Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:06:18.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 97397 9.20 MB/sec execute 137 sec latency 65.861 ms 2024-09-17T01:06:18.516 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 podman[65412]: 2024-09-17 01:06:18.1962929 +0000 UTC m=+0.019189762 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:06:18.516 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 podman[65412]: 2024-09-17 01:06:18.361581479 +0000 UTC m=+0.184478312 container create d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True) 2024-09-17T01:06:18.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 podman[65412]: 2024-09-17 01:06:18.756547957 +0000 UTC m=+0.579444808 container init d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:06:18.772 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 podman[65412]: 2024-09-17 01:06:18.765461024 +0000 UTC m=+0.588357861 container start d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T01:06:19.021 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:06:19.021 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T01:06:19.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: pidfile_write: ignore empty --pid-file 2024-09-17T01:06:19.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: load: jerasure load: lrc 2024-09-17T01:06:19.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: RocksDB version: 7.9.2 2024-09-17T01:06:19.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Git sha 0 2024-09-17T01:06:19.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T01:06:19.022 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: DB SUMMARY 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: DB Session ID: FGPU3MH2EDBYCQAQEX9J 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi033/store.db dir, Total Num: 1, files: 000030.sst 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi033/store.db: 000028.log size: 1210744 ; 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:06:19.023 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:06:19.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:06:19.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T01:06:19.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:06:19.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T01:06:19.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.env: 0x55e49d007940 2024-09-17T01:06:19.024 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T01:06:19.025 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.info_log: 0x55e49e3d38c0 2024-09-17T01:06:19.025 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:06:19.025 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.statistics: (nil) 2024-09-17T01:06:19.025 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.use_fsync: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:06:19.026 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:06:19.027 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.db_log_dir: 2024-09-17T01:06:19.027 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.wal_dir: 2024-09-17T01:06:19.027 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:06:19.028 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:06:19.028 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:06:19.028 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:06:19.028 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:06:19.028 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:06:19.028 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.write_buffer_manager: 0x55e49e3e2dc0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.unordered_write: 0 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:06:19.029 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.row_cache: None 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.wal_filter: None 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.wal_compression: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:06:19.030 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:06:19.031 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_open_files: -1 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Compression algorithms supported: 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kZSTD supported: 0 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:06:19.032 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi033/store.db/MANIFEST-000015 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.merge_operator: 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_filter: None 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:06:19.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e49e3d3520) 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: index_type: 0 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: data_block_index_type: 0 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: index_shortening: 1 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: checksum: 4 2024-09-17T01:06:19.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: no_block_cache: 0 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache: 0x55e49e3c71f0 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache_options: 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: capacity : 536870912 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: num_shard_bits : 4 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: strict_capacity_limit : 0 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_cache_compressed: (nil) 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: persistent_cache: (nil) 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_size: 4096 2024-09-17T01:06:19.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_size_deviation: 10 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_restart_interval: 16 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: index_block_restart_interval: 1 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: metadata_block_size: 4096 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: partition_filters: 0 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: use_delta_encoding: 1 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: filter_policy: bloomfilter 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: whole_key_filtering: 1 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: verify_compression: 0 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: format_version: 5 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: enable_index_compression: 1 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: block_align: 0 2024-09-17T01:06:19.036 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: max_auto_readahead_size: 262144 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: prepopulate_block_cache: 0 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: initial_auto_readahead_size: 8192 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression: NoCompression 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.num_levels: 7 2024-09-17T01:06:19.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:06:19.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:06:19.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:06:19.039 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:06:19.040 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:06:19.040 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:06:19.040 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:06:19.040 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:06:19.040 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:06:19.040 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:06:19.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:06:19.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:06:19.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:06:19.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:06:19.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T01:06:19.042 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:06:19.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:06:19.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:06:19.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:06:19.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:06:19.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:06:19.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:06:19.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:06:19.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:06:19.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:06:19.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.ttl: 2592000 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:06:19.046 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi033/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11868, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 270a5bd5-83dc-4816-9ae1-f053e3ce5b35 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535178798452, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-17T01:06:19.047 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535178802941, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1060919, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11862, "largest_seqno": 12562, "table_properties": {"data_size": 1057049, "index_size": 2062, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 7045, "raw_average_key_size": 23, "raw_value_size": 1050387, "raw_average_value_size": 3501, "num_data_blocks": 94, "num_entries": 300, "num_filter_entries": 300, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726535178, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "270a5bd5-83dc-4816-9ae1-f053e3ce5b35", "db_session_id": "FGPU3MH2EDBYCQAQEX9J", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535178803068, "job": 1, "event": "recovery_finished"} 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi033/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e49e3f4e00 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: DB pointer 0x55e49e4f0000 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** DB Stats ** 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:06:19.048 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: L0 1/0 1.01 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 399.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: L6 1/0 7.32 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Sum 2/0 8.33 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 399.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 399.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 399.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:06:19.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Cumulative compaction: 0.00 GB write, 140.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Interval compaction: 0.00 GB write, 140.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Block cache BinnedLRUCache@0x55e49e3c71f0#2 capacity: 512.00 MB usage: 3.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.33 KB,0.000444055%) Misc(1,0.00 KB,0%) 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: starting mon.smithi033 rank 0 at public addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] at bind addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi033 fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???) e2 preinit fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).mds e6 new map 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).mds e6 print_map 2024-09-17T01:06:19.051 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: e6 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: legacy client fscid: 1 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Filesystem 'foofs' (1) 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: fs_name foofs 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: epoch 4 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: created 2024-09-17T01:01:23.322312+0000 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: modified 2024-09-17T01:01:28.756774+0000 2024-09-17T01:06:19.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: tableserver 0 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: root 0 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: session_timeout 60 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: session_autoclose 300 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: max_file_size 1099511627776 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: max_xattr_size 65536 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: required_client_features {} 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: last_failure 0 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: last_failure_osd_epoch 0 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: max_mds 1 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: in 0 2024-09-17T01:06:19.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: up {0=14478} 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: failed 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: damaged 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: stopped 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: data_pools [3] 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: metadata_pool 2 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: inline_data disabled 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: balancer 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: bal_rank_mask -1 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: standby_count_wanted 1 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: qdb_cluster leader: 0 members: 2024-09-17T01:06:19.054 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: [mds.foofs.smithi033.gofagb{0:14478} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.33:6834/2928191622,v1:172.21.15.33:6835/2928191622] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Standby daemons: 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: [mds.foofs.smithi100.ssfebu{-1:24297} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 ceph-mon[65430]: mon.smithi033@-1(???).paxosservice(auth 1..28) refresh upgraded, format 0 -> 3 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 bash[65412]: d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 2024-09-17T01:06:19.055 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:18 smithi033 systemd[1]: Started Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:06:19.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 97649 9.20 MB/sec execute 138 sec latency 69.461 ms 2024-09-17T01:06:19.433 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:06:19.811 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:06:19.811 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 355 op/s 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 494 op/s 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: mon.smithi033 calling monitor election 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: mon.smithi033 is new leader, mons smithi033,smithi100 in quorum (ranks 0,1) 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: monmap epoch 2 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: last_changed 2024-09-17T00:56:20.484217+0000 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: created 2024-09-17T00:54:13.960446+0000 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: min_mon_release 17 (quincy) 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: election_strategy: 1 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.smithi033 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: 1: [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon.smithi100 2024-09-17T01:06:19.812 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:06:19.813 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:06:19.813 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: mgrmap e34: smithi033.illmog(active, since 53s), standbys: smithi100.qrwkie 2024-09-17T01:06:19.813 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: overall HEALTH_OK 2024-09-17T01:06:19.813 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: from='mgr.14740 ' entity='' 2024-09-17T01:06:19.813 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:19 smithi033 ceph-mon[65430]: mgrmap e35: smithi033.illmog(active, since 53s), standbys: smithi100.qrwkie 2024-09-17T01:06:19.963 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:06:19.964 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: from='mgr.14740 172.21.15.33:0/1297209853' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:06:19.964 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: pgmap v28: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.5 MiB/s wr, 355 op/s 2024-09-17T01:06:19.964 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: pgmap v29: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 494 op/s 2024-09-17T01:06:19.964 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: mon.smithi033 calling monitor election 2024-09-17T01:06:19.964 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: mon.smithi033 is new leader, mons smithi033,smithi100 in quorum (ranks 0,1) 2024-09-17T01:06:19.964 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: monmap epoch 2 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: last_changed 2024-09-17T00:56:20.484217+0000 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: created 2024-09-17T00:54:13.960446+0000 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: min_mon_release 17 (quincy) 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: election_strategy: 1 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.smithi033 2024-09-17T01:06:19.965 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: 1: [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon.smithi100 2024-09-17T01:06:19.966 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:06:19.966 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T01:06:19.966 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: mgrmap e34: smithi033.illmog(active, since 53s), standbys: smithi100.qrwkie 2024-09-17T01:06:19.966 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: overall HEALTH_OK 2024-09-17T01:06:19.966 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: from='mgr.14740 ' entity='' 2024-09-17T01:06:19.966 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:19 smithi100 ceph-mon[25838]: mgrmap e35: smithi033.illmog(active, since 53s), standbys: smithi100.qrwkie 2024-09-17T01:06:20.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 97915 9.19 MB/sec execute 139 sec latency 74.961 ms 2024-09-17T01:06:21.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 98258 9.14 MB/sec execute 140 sec latency 71.522 ms 2024-09-17T01:06:21.591 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:21 smithi033 ceph-mon[65430]: mgrmap e36: smithi033.illmog(active, since 54s), standbys: smithi100.qrwkie 2024-09-17T01:06:21.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:21 smithi100 ceph-mon[25838]: mgrmap e36: smithi033.illmog(active, since 54s), standbys: smithi100.qrwkie 2024-09-17T01:06:22.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 98535 9.10 MB/sec execute 141 sec latency 57.467 ms 2024-09-17T01:06:23.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 98829 9.12 MB/sec execute 142 sec latency 46.506 ms 2024-09-17T01:06:24.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 99227 9.15 MB/sec execute 143 sec latency 46.611 ms 2024-09-17T01:06:25.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 99597 9.19 MB/sec execute 144 sec latency 49.260 ms 2024-09-17T01:06:25.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:25 smithi033 ceph-mon[65430]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:06:25.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:25 smithi033 ceph-mon[65430]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:06:25.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:25 smithi033 ceph-mon[65430]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:06:25.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:25 smithi033 ceph-mon[65430]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:06:25.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:25 smithi033 ceph-mon[65430]: Standby manager daemon smithi100.qrwkie restarted 2024-09-17T01:06:25.443 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:25 smithi033 ceph-mon[65430]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:06:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/crt"}]: dispatch 2024-09-17T01:06:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T01:06:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi100.qrwkie/key"}]: dispatch 2024-09-17T01:06:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:25 smithi100 ceph-mon[25838]: from='mgr.? 172.21.15.100:0/2761835228' entity='mgr.smithi100.qrwkie' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T01:06:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:25 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie restarted 2024-09-17T01:06:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:25 smithi100 ceph-mon[25838]: Standby manager daemon smithi100.qrwkie started 2024-09-17T01:06:26.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 99978 9.21 MB/sec execute 145 sec latency 44.818 ms 2024-09-17T01:06:26.340 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:26 smithi033 ceph-mon[65430]: mgrmap e37: smithi033.illmog(active, since 59s), standbys: smithi100.qrwkie 2024-09-17T01:06:26.626 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:26 smithi100 ceph-mon[25838]: mgrmap e37: smithi033.illmog(active, since 59s), standbys: smithi100.qrwkie 2024-09-17T01:06:27.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 100154 9.17 MB/sec execute 146 sec latency 71.846 ms 2024-09-17T01:06:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: Active manager daemon smithi033.illmog restarted 2024-09-17T01:06:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: Activating manager daemon smithi033.illmog 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: mgrmap e38: smithi033.illmog(active, starting, since 0.00872434s), standbys: smithi100.qrwkie 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:06:27.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: Manager daemon smithi033.illmog is now available 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:27.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:27.366 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:06:27.366 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: Active manager daemon smithi033.illmog restarted 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: Activating manager daemon smithi033.illmog 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: mgrmap e38: smithi033.illmog(active, starting, since 0.00872434s), standbys: smithi100.qrwkie 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi033.illmog", "id": "smithi033.illmog"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr metadata", "who": "smithi100.qrwkie", "id": "smithi100.qrwkie"}]: dispatch 2024-09-17T01:06:27.641 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: Manager daemon smithi033.illmog is now available 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/mirror_snapshot_schedule"}]: dispatch 2024-09-17T01:06:27.642 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:27 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi033.illmog/trash_purge_schedule"}]: dispatch 2024-09-17T01:06:28.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 100373 9.15 MB/sec execute 147 sec latency 85.152 ms 2024-09-17T01:06:28.539 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:06:28.539 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (2m) 18s ago 10m 17.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:06:28.539 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (10m) 18s ago 10m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:06:28.539 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (10m) 22s ago 10m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:06:28.539 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (2m) 18s ago 10m 44.4M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:06:28.540 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (5m) 18s ago 5m 35.3M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:06:28.540 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (4m) 22s ago 4m 13.7M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:06:28.540 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (79s) 18s ago 12m 494M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:06:28.540 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (27s) 22s ago 10m 94.4M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:06:28.540 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (12m) 18s ago 12m 52.1M 2048M 17.2.0 e1d6a67b021e 48394a6f2f5f 2024-09-17T01:06:28.540 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (10m) 22s ago 10m 45.4M 2048M 17.2.0 e1d6a67b021e f01d7a13b176 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (4m) 18s ago 4m 265M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (4m) 22s ago 4m 84.2M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (117s) 18s ago 10m 8040k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (87s) 22s ago 9m 7478k - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (9m) 18s ago 9m 386M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (8m) 18s ago 8m 291M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:06:28.541 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (8m) 18s ago 8m 246M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:06:28.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (8m) 18s ago 8m 352M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:06:28.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (7m) 22s ago 7m 354M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:06:28.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (7m) 22s ago 7m 380M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:06:28.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (6m) 22s ago 6m 315M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:06:28.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (6m) 22s ago 6m 449M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:06:28.542 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (33s) 18s ago 9m 46.9M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:06:29.127 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:28 smithi033 ceph-mon[65430]: from='client.34100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:06:29.127 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:28 smithi033 ceph-mon[65430]: mgrmap e39: smithi033.illmog(active, since 1.02224s), standbys: smithi100.qrwkie 2024-09-17T01:06:29.128 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:28 smithi033 ceph-mon[65430]: pgmap v3: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:29.128 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:28 smithi033 ceph-mon[65430]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:06:29.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:28 smithi100 ceph-mon[25838]: from='client.34100 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:06:29.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:28 smithi100 ceph-mon[25838]: mgrmap e39: smithi033.illmog(active, since 1.02224s), standbys: smithi100.qrwkie 2024-09-17T01:06:29.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:28 smithi100 ceph-mon[25838]: pgmap v3: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:29.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:28 smithi100 ceph-mon[25838]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:06:29.256 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:06:29.257 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:06:29.258 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-17T01:06:29.258 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:06:29.258 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:06:29.258 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:06:29.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 100593 9.11 MB/sec execute 148 sec latency 71.423 ms 2024-09-17T01:06:29.784 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:06:29.784 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:06:29.784 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:06:29.784 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:06:29.784 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:06:29.784 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-17T01:06:29.785 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:06:29.785 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T01:06:29.785 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-17T01:06:29.785 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:06:29.785 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:06:30.140 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:29 smithi033 ceph-mon[65430]: pgmap v4: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:30.140 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:29 smithi033 ceph-mon[65430]: mgrmap e40: smithi033.illmog(active, since 2s), standbys: smithi100.qrwkie 2024-09-17T01:06:30.140 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:29 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/512414306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:06:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:29 smithi100 ceph-mon[25838]: pgmap v4: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:29 smithi100 ceph-mon[25838]: mgrmap e40: smithi033.illmog(active, since 2s), standbys: smithi100.qrwkie 2024-09-17T01:06:30.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:29 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/512414306' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:06:30.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 100824 9.08 MB/sec execute 149 sec latency 70.954 ms 2024-09-17T01:06:30.567 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:30 smithi033 ceph-mon[65430]: [17/Sep/2024:01:06:30] ENGINE Bus STARTING 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:30 smithi033 ceph-mon[65430]: [17/Sep/2024:01:06:30] ENGINE Serving on http://172.21.15.33:8765 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:30 smithi033 ceph-mon[65430]: [17/Sep/2024:01:06:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:30 smithi033 ceph-mon[65430]: [17/Sep/2024:01:06:30] ENGINE Bus STARTED 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:30 smithi033 ceph-mon[65430]: [17/Sep/2024:01:06:30] ENGINE Error in HTTPServer.serve 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: Traceback (most recent call last): 2024-09-17T01:06:31.165 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._run(expiration_interval) 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: s = self.context.wrap_socket( 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: return self.sslsocket_class._create( 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self.do_handshake() 2024-09-17T01:06:31.166 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:06:31.167 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: self._sslobj.do_handshake() 2024-09-17T01:06:31.167 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:06:31.167 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:30 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/792566584' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:06:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:06:30] ENGINE Bus STARTING 2024-09-17T01:06:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:06:30] ENGINE Serving on http://172.21.15.33:8765 2024-09-17T01:06:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:06:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-17T01:06:31.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:06:30] ENGINE Bus STARTED 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:30 smithi100 ceph-mon[25838]: [17/Sep/2024:01:06:30] ENGINE Error in HTTPServer.serve 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Traceback (most recent call last): 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._connections.run(self.expiration_interval) 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._run(expiration_interval) 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: s = self.context.wrap_socket( 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T01:06:31.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: return self.sslsocket_class._create( 2024-09-17T01:06:31.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T01:06:31.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self.do_handshake() 2024-09-17T01:06:31.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T01:06:31.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: self._sslobj.do_handshake() 2024-09-17T01:06:31.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T01:06:31.229 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:30 smithi100 ceph-mon[25838]: from='client.? 172.21.15.33:0/792566584' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:06:31.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 101114 9.07 MB/sec execute 150 sec latency 50.779 ms 2024-09-17T01:06:32.184 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:31 smithi033 ceph-mon[65430]: pgmap v5: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:32.184 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:31 smithi033 ceph-mon[65430]: mgrmap e41: smithi033.illmog(active, since 4s), standbys: smithi100.qrwkie 2024-09-17T01:06:32.184 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:31 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:32.184 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:31 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:31 smithi100 ceph-mon[25838]: pgmap v5: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:31 smithi100 ceph-mon[25838]: mgrmap e41: smithi033.illmog(active, since 4s), standbys: smithi100.qrwkie 2024-09-17T01:06:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:31 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:31 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:32.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 101479 9.07 MB/sec execute 151 sec latency 52.847 ms 2024-09-17T01:06:33.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 101837 9.05 MB/sec execute 152 sec latency 64.290 ms 2024-09-17T01:06:33.705 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:33 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:33.705 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:33 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:33 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:33 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:34.300 INFO:teuthology.orchestra.run.smithi033.stdout: 5 102246 9.05 MB/sec execute 153 sec latency 87.762 ms 2024-09-17T01:06:34.713 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:34 smithi033 ceph-mon[65430]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:34.726 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:34 smithi100 ceph-mon[25838]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-17T01:06:35.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 102562 9.08 MB/sec execute 154 sec latency 71.869 ms 2024-09-17T01:06:36.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 102979 9.13 MB/sec execute 155 sec latency 50.629 ms 2024-09-17T01:06:36.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:36 smithi033 ceph-mon[65430]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 2.6 MiB/s wr, 326 op/s 2024-09-17T01:06:36.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:36 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:36.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:36 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:36 smithi100 ceph-mon[25838]: pgmap v7: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 2.6 MiB/s wr, 326 op/s 2024-09-17T01:06:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:36 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:36 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:37.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 103336 9.14 MB/sec execute 156 sec latency 46.597 ms 2024-09-17T01:06:38.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 253 op/s 2024-09-17T01:06:38.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.044 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:38.045 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:06:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: pgmap v8: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 253 op/s 2024-09-17T01:06:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:38.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:38.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:37 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:06:38.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 103656 9.14 MB/sec execute 157 sec latency 54.882 ms 2024-09-17T01:06:39.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:38 smithi033 ceph-mon[65430]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T01:06:39.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:38 smithi033 ceph-mon[65430]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T01:06:39.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:38 smithi033 ceph-mon[65430]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:06:39.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:38 smithi033 ceph-mon[65430]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:06:39.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:38 smithi033 ceph-mon[65430]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:06:39.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:38 smithi033 ceph-mon[65430]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:06:39.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:38 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-17T01:06:39.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:38 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.conf 2024-09-17T01:06:39.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:38 smithi100 ceph-mon[25838]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:06:39.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:38 smithi100 ceph-mon[25838]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T01:06:39.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:38 smithi100 ceph-mon[25838]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:06:39.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:38 smithi100 ceph-mon[25838]: Updating smithi100:/etc/ceph/ceph.client.admin.keyring 2024-09-17T01:06:39.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 103827 9.09 MB/sec execute 158 sec latency 72.548 ms 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.4 MiB/s wr, 447 op/s 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:06:40.072 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:39 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:06:40.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: Updating smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:06:40.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: pgmap v9: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.4 MiB/s wr, 447 op/s 2024-09-17T01:06:40.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: Updating smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:06:40.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T01:06:40.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 104046 9.07 MB/sec execute 159 sec latency 63.184 ms 2024-09-17T01:06:41.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 104328 9.04 MB/sec execute 160 sec latency 58.546 ms 2024-09-17T01:06:41.884 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: Upgrade: Updating mon.smithi100 2024-09-17T01:06:41.884 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:41.884 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:06:41.884 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:06:41.885 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:41.885 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: Deploying daemon mon.smithi100 on smithi100 2024-09-17T01:06:41.885 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:41 smithi033 ceph-mon[65430]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-09-17T01:06:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: Upgrade: Updating mon.smithi100 2024-09-17T01:06:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:06:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:06:42.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:42.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: Deploying daemon mon.smithi100 on smithi100 2024-09-17T01:06:42.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:41 smithi100 ceph-mon[25838]: pgmap v10: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.0 MiB/s wr, 409 op/s 2024-09-17T01:06:42.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 104833 9.08 MB/sec execute 161 sec latency 61.886 ms 2024-09-17T01:06:42.843 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:42 smithi100 systemd[1]: Stopping Ceph mon.smithi100 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:06:43.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100[25824]: 2024-09-17T01:06:42.843+0000 7f8a45b21700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi100 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:06:43.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100[25824]: 2024-09-17T01:06:42.843+0000 7f8a45b21700 -1 mon.smithi100@1(peon) e2 *** Got Signal Terminated *** 2024-09-17T01:06:43.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:42 smithi100 podman[51548]: 2024-09-17 01:06:42.93619719 +0000 UTC m=+0.199882323 container died f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.expose-services=, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-type=git, name=centos-stream, release=754, io.buildah.version=1.19.8) 2024-09-17T01:06:43.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 105278 9.08 MB/sec execute 162 sec latency 45.039 ms 2024-09-17T01:06:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 podman[51548]: 2024-09-17 01:06:43.329039781 +0000 UTC m=+0.592724917 container cleanup f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T01:06:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 bash[51548]: ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100 2024-09-17T01:06:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 podman[51562]: 2024-09-17 01:06:43.581221486 +0000 UTC m=+0.642190801 container remove f01d7a13b176cf966a09c06b6f9c6dea19420af9aa32ccaf86089879c2649cf2 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.buildah.version=1.19.8, version=8, RELEASE=HEAD, release=754, ceph=True, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T01:06:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi100.service: Deactivated successfully. 2024-09-17T01:06:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 systemd[1]: Stopped Ceph mon.smithi100 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:06:43.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi100.service: Consumed 6.665s CPU time. 2024-09-17T01:06:44.271 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:43 smithi100 systemd[1]: Starting Ceph mon.smithi100 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:06:44.271 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 podman[51682]: 2024-09-17 01:06:44.144954405 +0000 UTC m=+0.020295954 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:06:44.271 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 podman[51682]: 2024-09-17 01:06:44.274073588 +0000 UTC m=+0.149415146 container create 6dbe4cef4690a7f56075539214087ca268b61596a9034260391fd925731ac5f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:06:44.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 105605 9.04 MB/sec execute 163 sec latency 59.539 ms 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 podman[51682]: 2024-09-17 01:06:44.535782142 +0000 UTC m=+0.411123700 container init 6dbe4cef4690a7f56075539214087ca268b61596a9034260391fd925731ac5f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2) 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 podman[51682]: 2024-09-17 01:06:44.540720969 +0000 UTC m=+0.416062522 container start 6dbe4cef4690a7f56075539214087ca268b61596a9034260391fd925731ac5f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: pidfile_write: ignore empty --pid-file 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: load: jerasure load: lrc 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: RocksDB version: 7.9.2 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Git sha 0 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T01:06:44.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: DB SUMMARY 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: DB Session ID: LVY10YRE0XM1OK3JAM7A 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: CURRENT file: CURRENT 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: MANIFEST file: MANIFEST-000009 size: 904 Bytes 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi100/store.db dir, Total Num: 1, files: 000024.sst 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi100/store.db: 000022.log size: 5046175 ; 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.error_if_exists: 0 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.create_if_missing: 0 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.paranoid_checks: 1 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T01:06:44.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.env: 0x561656482940 2024-09-17T01:06:44.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T01:06:44.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.info_log: 0x56165815d8c0 2024-09-17T01:06:44.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T01:06:44.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.statistics: (nil) 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.use_fsync: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_log_file_size: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.allow_fallocate: 1 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.use_direct_reads: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T01:06:44.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.db_log_dir: 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.wal_dir: 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.write_buffer_manager: 0x56165816cdc0 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T01:06:44.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.unordered_write: 0 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.row_cache: None 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.wal_filter: None 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T01:06:44.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.two_write_queues: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.wal_compression: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.atomic_flush: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.log_readahead_size: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T01:06:44.984 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_background_jobs: 2 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_background_compactions: -1 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_subcompactions: 1 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T01:06:44.985 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_open_files: -1 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_background_flushes: -1 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Compression algorithms supported: 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kZSTD supported: 0 2024-09-17T01:06:44.986 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kXpressCompression supported: 0 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kBZip2Compression supported: 0 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kLZ4Compression supported: 1 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kZlibCompression supported: 1 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: kSnappyCompression supported: 1 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi100/store.db/MANIFEST-000009 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.merge_operator: 2024-09-17T01:06:44.987 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_filter: None 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56165815d520) 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: cache_index_and_filter_blocks: 1 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: pin_top_level_index_and_filter: 1 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: index_type: 0 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: data_block_index_type: 0 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: index_shortening: 1 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T01:06:44.988 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: checksum: 4 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: no_block_cache: 0 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache: 0x5616581511f0 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache_name: BinnedLRUCache 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache_options: 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: capacity : 536870912 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: num_shard_bits : 4 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: strict_capacity_limit : 0 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: high_pri_pool_ratio: 0.000 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_cache_compressed: (nil) 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: persistent_cache: (nil) 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_size: 4096 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_size_deviation: 10 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_restart_interval: 16 2024-09-17T01:06:44.989 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: index_block_restart_interval: 1 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: metadata_block_size: 4096 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: partition_filters: 0 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: use_delta_encoding: 1 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: filter_policy: bloomfilter 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: whole_key_filtering: 1 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: verify_compression: 0 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: read_amp_bytes_per_bit: 0 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: format_version: 5 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: enable_index_compression: 1 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: block_align: 0 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: max_auto_readahead_size: 262144 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: prepopulate_block_cache: 0 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: initial_auto_readahead_size: 8192 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T01:06:44.990 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression: NoCompression 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.num_levels: 7 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T01:06:44.991 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T01:06:44.992 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T01:06:44.993 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T01:06:44.994 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T01:06:44.994 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T01:06:44.994 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T01:06:44.994 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T01:06:44.994 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T01:06:44.994 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T01:06:44.995 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.inplace_update_support: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.bloom_locality: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.max_successive_merges: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T01:06:44.996 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.ttl: 2592000 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.enable_blob_files: false 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.min_blob_size: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T01:06:44.997 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi100/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12282, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6a270a39-4d88-40c4-bc0f-cd3b351d8e2f 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535204577343, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535204591090, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 3106279, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12284, "largest_seqno": 13579, "table_properties": {"data_size": 3100241, "index_size": 3649, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1349, "raw_key_size": 12292, "raw_average_key_size": 23, "raw_value_size": 3088730, "raw_average_value_size": 5883, "num_data_blocks": 168, "num_entries": 525, "num_filter_entries": 525, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726535204, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6a270a39-4d88-40c4-bc0f-cd3b351d8e2f", "db_session_id": "LVY10YRE0XM1OK3JAM7A", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726535204591280, "job": 1, "event": "recovery_finished"} 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-17T01:06:44.998 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi100/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56165817ee00 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: DB pointer 0x56165827a000 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** DB Stats ** 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T01:06:44.999 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** Compaction Stats [default] ** 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: L0 1/0 2.96 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 568.8 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: L6 1/0 7.32 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Sum 2/0 10.28 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 568.8 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 568.8 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** Compaction Stats [default] ** 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 568.8 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T01:06:45.000 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Cumulative compaction: 0.00 GB write, 126.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Interval compaction: 0.00 GB write, 126.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Block cache BinnedLRUCache@0x5616581511f0#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.001 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: starting mon.smithi100 rank 1 at public addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] at bind addrs [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi100 fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???) e2 preinit fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).mds e6 new map 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).mds e6 print_map 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: e6 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: legacy client fscid: 1 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Filesystem 'foofs' (1) 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: fs_name foofs 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: epoch 4 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T01:06:45.002 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: created 2024-09-17T01:01:23.322312+0000 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: modified 2024-09-17T01:01:28.756774+0000 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: tableserver 0 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: root 0 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: session_timeout 60 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: session_autoclose 300 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: max_file_size 1099511627776 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: max_xattr_size 65536 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: required_client_features {} 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: last_failure 0 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: last_failure_osd_epoch 0 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: max_mds 1 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: in 0 2024-09-17T01:06:45.003 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: up {0=14478} 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: failed 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: damaged 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: stopped 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: data_pools [3] 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: metadata_pool 2 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: inline_data disabled 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: balancer 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: bal_rank_mask -1 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: standby_count_wanted 1 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: qdb_cluster leader: 0 members: 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: [mds.foofs.smithi033.gofagb{0:14478} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.33:6834/2928191622,v1:172.21.15.33:6835/2928191622] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.004 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: Standby daemons: 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout: [mds.foofs.smithi100.ssfebu{-1:24297} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 ceph-mon[51701]: mon.smithi100@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 bash[51682]: 6dbe4cef4690a7f56075539214087ca268b61596a9034260391fd925731ac5f7 2024-09-17T01:06:45.005 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:44 smithi100 systemd[1]: Started Ceph mon.smithi100 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:06:45.301 INFO:teuthology.orchestra.run.smithi033.stdout: 5 105915 9.06 MB/sec execute 164 sec latency 50.901 ms 2024-09-17T01:06:46.255 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:46 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:46.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 106184 9.06 MB/sec execute 165 sec latency 55.545 ms 2024-09-17T01:06:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:46 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:47.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:06:47.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:06:47.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: mon.smithi100 calling monitor election 2024-09-17T01:06:47.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: mon.smithi033 calling monitor election 2024-09-17T01:06:47.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: mon.smithi033 is new leader, mons smithi033,smithi100 in quorum (ranks 0,1) 2024-09-17T01:06:47.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: monmap epoch 3 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: last_changed 2024-09-17T01:06:46.048808+0000 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: created 2024-09-17T00:54:13.960446+0000 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: min_mon_release 19 (squid) 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: election_strategy: 1 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.smithi033 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: 1: [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon.smithi100 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: mgrmap e41: smithi033.illmog(active, since 19s), standbys: smithi100.qrwkie 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: overall HEALTH_OK 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:47.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:47.273 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:47.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 106554 9.09 MB/sec execute 166 sec latency 70.481 ms 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi033"}]: dispatch 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mon metadata", "id": "smithi100"}]: dispatch 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: mon.smithi100 calling monitor election 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: mon.smithi033 calling monitor election 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: mon.smithi033 is new leader, mons smithi033,smithi100 in quorum (ranks 0,1) 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: monmap epoch 3 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: fsid 47eab686-748f-11ef-bceb-c7b262605968 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: last_changed 2024-09-17T01:06:46.048808+0000 2024-09-17T01:06:47.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: created 2024-09-17T00:54:13.960446+0000 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: min_mon_release 19 (squid) 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: election_strategy: 1 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.smithi033 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: 1: [v2:172.21.15.100:3300/0,v1:172.21.15.100:6789/0] mon.smithi100 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: mgrmap e41: smithi033.illmog(active, since 19s), standbys: smithi100.qrwkie 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: overall HEALTH_OK 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:47.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:06:48.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:48 smithi100 ceph-mon[51701]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 411 op/s 2024-09-17T01:06:48.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 106888 9.10 MB/sec execute 167 sec latency 64.387 ms 2024-09-17T01:06:48.349 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:48 smithi033 ceph-mon[65430]: pgmap v13: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.2 MiB/s wr, 411 op/s 2024-09-17T01:06:49.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 107090 9.09 MB/sec execute 168 sec latency 68.089 ms 2024-09-17T01:06:50.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 107219 9.06 MB/sec execute 169 sec latency 80.261 ms 2024-09-17T01:06:50.666 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:50 smithi033 ceph-mon[65430]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 604 op/s 2024-09-17T01:06:50.666 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:50 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:50.666 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:50 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:50 smithi100 ceph-mon[51701]: pgmap v14: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.3 MiB/s wr, 604 op/s 2024-09-17T01:06:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:50 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:50 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:51.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 107422 9.03 MB/sec execute 170 sec latency 64.278 ms 2024-09-17T01:06:52.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 107673 9.00 MB/sec execute 171 sec latency 58.807 ms 2024-09-17T01:06:52.484 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:52 smithi033 ceph-mon[65430]: pgmap v15: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 384 op/s 2024-09-17T01:06:52.484 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:52.484 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:52.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:52 smithi100 ceph-mon[51701]: pgmap v15: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 384 op/s 2024-09-17T01:06:52.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:52.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:53.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 107966 8.99 MB/sec execute 172 sec latency 57.934 ms 2024-09-17T01:06:54.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 108269 9.00 MB/sec execute 173 sec latency 57.113 ms 2024-09-17T01:06:54.519 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:54 smithi033 ceph-mon[65430]: pgmap v16: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 384 op/s 2024-09-17T01:06:54.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:54 smithi100 ceph-mon[51701]: pgmap v16: 97 pgs: 97 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 384 op/s 2024-09-17T01:06:55.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 108645 8.99 MB/sec execute 174 sec latency 50.470 ms 2024-09-17T01:06:56.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:56.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:06:56.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.037 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: Reconfiguring mon.smithi033 (monmap changed)... 2024-09-17T01:06:56.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:06:56.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:06:56.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:56.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: Reconfiguring daemon mon.smithi033 on smithi033 2024-09-17T01:06:56.038 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:55 smithi033 ceph-mon[65430]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: Reconfiguring mon.smithi033 (monmap changed)... 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:06:56.129 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:56.130 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: Reconfiguring daemon mon.smithi033 on smithi033 2024-09-17T01:06:56.130 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:55 smithi100 ceph-mon[51701]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.3 MiB/s wr, 525 op/s 2024-09-17T01:06:56.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 109051 8.96 MB/sec execute 175 sec latency 54.942 ms 2024-09-17T01:06:57.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 109324 8.94 MB/sec execute 176 sec latency 42.583 ms 2024-09-17T01:06:58.077 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 333 op/s 2024-09-17T01:06:58.077 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:58.077 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:58.078 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:58.078 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: Reconfiguring mgr.smithi033.illmog (monmap changed)... 2024-09-17T01:06:58.078 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:06:58.078 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:06:58.078 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:58.078 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:57 smithi033 ceph-mon[65430]: Reconfiguring daemon mgr.smithi033.illmog on smithi033 2024-09-17T01:06:58.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 109690 8.98 MB/sec execute 177 sec latency 67.659 ms 2024-09-17T01:06:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 333 op/s 2024-09-17T01:06:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:06:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:06:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: Reconfiguring mgr.smithi033.illmog (monmap changed)... 2024-09-17T01:06:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi033.illmog", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:06:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:06:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:06:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:57 smithi100 ceph-mon[51701]: Reconfiguring daemon mgr.smithi033.illmog on smithi033 2024-09-17T01:06:59.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 110046 9.01 MB/sec execute 178 sec latency 52.717 ms 2024-09-17T01:06:59.857 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:06:59 smithi033 ceph-mon[65430]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.5 MiB/s wr, 551 op/s 2024-09-17T01:07:00.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:06:59 smithi100 ceph-mon[51701]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 5.5 MiB/s wr, 551 op/s 2024-09-17T01:07:00.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 110394 9.02 MB/sec execute 179 sec latency 44.030 ms 2024-09-17T01:07:01.167 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:07:01.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 110637 9.02 MB/sec execute 180 sec latency 61.203 ms 2024-09-17T01:07:01.649 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:01.649 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:01.649 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:01 smithi033 ceph-mon[65430]: Reconfiguring crash.smithi033 (monmap changed)... 2024-09-17T01:07:01.649 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:01.649 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:01.650 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:01 smithi033 ceph-mon[65430]: Reconfiguring daemon crash.smithi033 on smithi033 2024-09-17T01:07:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:01 smithi100 ceph-mon[51701]: Reconfiguring crash.smithi033 (monmap changed)... 2024-09-17T01:07:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:01.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:01.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:01 smithi100 ceph-mon[51701]: Reconfiguring daemon crash.smithi033 on smithi033 2024-09-17T01:07:02.033 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:07:02.033 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (2m) 25s ago 11m 17.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:07:02.033 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (11m) 25s ago 11m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:07:02.033 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (10m) 11s ago 10m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (2m) 25s ago 11m 44.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (5m) 25s ago 5m 35.7M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (5m) 11s ago 5m 14.0M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (113s) 25s ago 12m 473M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (60s) 11s ago 10m 438M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (43s) 25s ago 12m 31.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (17s) 11s ago 10m 22.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (5m) 25s ago 5m 267M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (5m) 11s ago 5m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (2m) 25s ago 11m 8216k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (2m) 11s ago 10m 8334k - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (9m) 25s ago 9m 423M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:07:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (9m) 25s ago 9m 313M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (9m) 25s ago 9m 249M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (8m) 25s ago 8m 380M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (8m) 11s ago 8m 384M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (7m) 11s ago 7m 422M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (7m) 11s ago 7m 349M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (7m) 11s ago 7m 495M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:07:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (67s) 25s ago 10m 51.3M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:07:02.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 110808 9.00 MB/sec execute 181 sec latency 70.079 ms 2024-09-17T01:07:02.567 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:07:02.567 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:07:02.567 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:07:02.567 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:07:02.568 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T01:07:02.569 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T01:07:02.569 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:07:02.569 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:07:02.705 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:02 smithi033 ceph-mon[65430]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 358 op/s 2024-09-17T01:07:02.705 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:02 smithi033 ceph-mon[65430]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:02.705 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:02 smithi033 ceph-mon[65430]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:02.705 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:02 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2053333444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:02.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:02 smithi100 ceph-mon[51701]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.4 MiB/s wr, 358 op/s 2024-09-17T01:07:02.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:02 smithi100 ceph-mon[51701]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:02.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:02 smithi100 ceph-mon[51701]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:02.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:02 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2053333444' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:03.154 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:07:03.154 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:07:03.154 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:07:03.155 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:07:03.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 111015 8.97 MB/sec execute 182 sec latency 64.299 ms 2024-09-17T01:07:03.800 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:07:03.888 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:03 smithi033 ceph-mon[65430]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:03.888 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:03 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:03.888 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:03 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:03.888 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:03 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:07:03.888 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:03 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:03.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:03 smithi100 ceph-mon[51701]: from='client.34160 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:03.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:03 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:03.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:03 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:03.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:03 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:07:03.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:03 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:04.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 111311 8.94 MB/sec execute 183 sec latency 64.060 ms 2024-09-17T01:07:04.642 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:04 smithi033 ceph-mon[65430]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T01:07:04.642 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:04 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.0 on smithi033 2024-09-17T01:07:04.642 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:04 smithi033 ceph-mon[65430]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 358 op/s 2024-09-17T01:07:04.643 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:04 smithi033 ceph-mon[65430]: from='client.44113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:04.643 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:04 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1369644444' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:07:04.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:04 smithi100 ceph-mon[51701]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T01:07:04.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:04 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.0 on smithi033 2024-09-17T01:07:04.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:04 smithi100 ceph-mon[51701]: pgmap v21: 97 pgs: 97 active+clean; 339 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.5 MiB/s wr, 358 op/s 2024-09-17T01:07:04.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:04 smithi100 ceph-mon[51701]: from='client.44113 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:04.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:04 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1369644444' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:07:05.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 111608 8.94 MB/sec execute 184 sec latency 55.938 ms 2024-09-17T01:07:06.302 INFO:teuthology.orchestra.run.smithi033.stdout: 5 111843 8.92 MB/sec execute 185 sec latency 124.796 ms 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 518 op/s 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:06.704 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:06 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.1 on smithi033 2024-09-17T01:07:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: pgmap v22: 97 pgs: 97 active+clean; 348 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.5 MiB/s wr, 518 op/s 2024-09-17T01:07:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:06.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T01:07:06.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:07:06.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:06.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:06 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.1 on smithi033 2024-09-17T01:07:07.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 112292 8.93 MB/sec execute 186 sec latency 62.003 ms 2024-09-17T01:07:08.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 112740 8.91 MB/sec execute 187 sec latency 63.426 ms 2024-09-17T01:07:08.748 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:08 smithi033 ceph-mon[65430]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-17T01:07:08.748 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:08 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:08.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:08 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:08.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:08 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:07:08.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:08 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:08 smithi100 ceph-mon[51701]: pgmap v23: 97 pgs: 97 active+clean; 348 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-17T01:07:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:08 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:08 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:08 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:07:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:08 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:09.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 113042 8.91 MB/sec execute 188 sec latency 54.261 ms 2024-09-17T01:07:09.798 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:09 smithi033 ceph-mon[65430]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T01:07:09.798 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:09 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.2 on smithi033 2024-09-17T01:07:09.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:09 smithi100 ceph-mon[51701]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T01:07:09.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:09 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.2 on smithi033 2024-09-17T01:07:10.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 113417 8.96 MB/sec execute 189 sec latency 53.586 ms 2024-09-17T01:07:10.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:10 smithi033 ceph-mon[65430]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.4 MiB/s wr, 578 op/s 2024-09-17T01:07:10.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:10.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:10.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:07:10.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:10 smithi100 ceph-mon[51701]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.4 MiB/s wr, 578 op/s 2024-09-17T01:07:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:07:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:11.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 113866 8.97 MB/sec execute 190 sec latency 40.190 ms 2024-09-17T01:07:11.879 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:11 smithi033 ceph-mon[65430]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T01:07:11.879 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:11 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.3 on smithi033 2024-09-17T01:07:11.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:11 smithi100 ceph-mon[51701]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T01:07:11.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:11 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.3 on smithi033 2024-09-17T01:07:12.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 114126 8.99 MB/sec execute 191 sec latency 51.192 ms 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 361 op/s 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: Reconfiguring mds.foofs.smithi033.gofagb (monmap changed)... 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:12.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:12 smithi033 ceph-mon[65430]: Reconfiguring daemon mds.foofs.smithi033.gofagb on smithi033 2024-09-17T01:07:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 361 op/s 2024-09-17T01:07:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: Reconfiguring mds.foofs.smithi033.gofagb (monmap changed)... 2024-09-17T01:07:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:07:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:12 smithi100 ceph-mon[51701]: Reconfiguring daemon mds.foofs.smithi033.gofagb on smithi033 2024-09-17T01:07:13.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 114253 8.97 MB/sec execute 192 sec latency 59.211 ms 2024-09-17T01:07:13.948 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:13 smithi033 ceph-mon[65430]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 361 op/s 2024-09-17T01:07:13.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:13 smithi100 ceph-mon[51701]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 361 op/s 2024-09-17T01:07:14.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 114450 8.94 MB/sec execute 193 sec latency 80.745 ms 2024-09-17T01:07:15.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 114632 8.91 MB/sec execute 194 sec latency 79.984 ms 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: Reconfiguring crash.smithi100 (monmap changed)... 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: Reconfiguring daemon crash.smithi100 on smithi100 2024-09-17T01:07:16.005 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:15 smithi033 ceph-mon[65430]: pgmap v27: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 554 op/s 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: Reconfiguring crash.smithi100 (monmap changed)... 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: Reconfiguring daemon crash.smithi100 on smithi100 2024-09-17T01:07:16.191 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:15 smithi100 ceph-mon[51701]: pgmap v27: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.9 MiB/s wr, 554 op/s 2024-09-17T01:07:16.303 INFO:teuthology.orchestra.run.smithi033.stdout: 5 114904 8.90 MB/sec execute 195 sec latency 75.060 ms 2024-09-17T01:07:17.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:17.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:17.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: Reconfiguring mgr.smithi100.qrwkie (monmap changed)... 2024-09-17T01:07:17.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:07:17.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:07:17.291 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:17.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:17 smithi033 ceph-mon[65430]: Reconfiguring daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T01:07:17.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 115339 8.90 MB/sec execute 196 sec latency 72.289 ms 2024-09-17T01:07:17.601 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:17.602 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:17.602 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: Reconfiguring mgr.smithi100.qrwkie (monmap changed)... 2024-09-17T01:07:17.602 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi100.qrwkie", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T01:07:17.602 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T01:07:17.602 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:17.602 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:17 smithi100 ceph-mon[51701]: Reconfiguring daemon mgr.smithi100.qrwkie on smithi100 2024-09-17T01:07:18.295 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:18 smithi033 ceph-mon[65430]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 394 op/s 2024-09-17T01:07:18.295 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:18.295 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:18.296 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:07:18.296 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:07:18.296 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:18.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 116107 8.93 MB/sec execute 197 sec latency 50.568 ms 2024-09-17T01:07:18.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:18 smithi100 ceph-mon[51701]: pgmap v28: 97 pgs: 97 active+clean; 345 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.9 MiB/s wr, 394 op/s 2024-09-17T01:07:18.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:18.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:18.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T01:07:18.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T01:07:18.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:19.285 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:19 smithi100 ceph-mon[51701]: Reconfiguring mon.smithi100 (monmap changed)... 2024-09-17T01:07:19.286 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:19 smithi100 ceph-mon[51701]: Reconfiguring daemon mon.smithi100 on smithi100 2024-09-17T01:07:19.303 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:19 smithi033 ceph-mon[65430]: Reconfiguring mon.smithi100 (monmap changed)... 2024-09-17T01:07:19.303 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:19 smithi033 ceph-mon[65430]: Reconfiguring daemon mon.smithi100 on smithi100 2024-09-17T01:07:19.311 INFO:teuthology.orchestra.run.smithi033.stdout: 5 116571 8.96 MB/sec execute 198 sec latency 52.599 ms 2024-09-17T01:07:20.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 117045 8.99 MB/sec execute 199 sec latency 51.072 ms 2024-09-17T01:07:20.313 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:20 smithi033 ceph-mon[65430]: pgmap v29: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-09-17T01:07:20.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:20.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:20.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:07:20.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:20 smithi100 ceph-mon[51701]: pgmap v29: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.7 MiB/s wr, 598 op/s 2024-09-17T01:07:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:07:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:21.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 117477 9.02 MB/sec execute 200 sec latency 34.823 ms 2024-09-17T01:07:21.337 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:21 smithi033 ceph-mon[65430]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T01:07:21.337 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:21 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.4 on smithi100 2024-09-17T01:07:21.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:21 smithi100 ceph-mon[51701]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T01:07:21.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:21 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.4 on smithi100 2024-09-17T01:07:22.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 117776 9.03 MB/sec execute 201 sec latency 46.672 ms 2024-09-17T01:07:22.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: pgmap v30: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 396 op/s 2024-09-17T01:07:22.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:22.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:22.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T01:07:22.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:07:22.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:22.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:22 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.5 on smithi100 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: pgmap v30: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 396 op/s 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:22 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.5 on smithi100 2024-09-17T01:07:23.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 117974 9.02 MB/sec execute 202 sec latency 73.614 ms 2024-09-17T01:07:24.153 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: pgmap v31: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 396 op/s 2024-09-17T01:07:24.154 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:24.154 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:24.154 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T01:07:24.154 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:07:24.154 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:24.154 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:24 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.6 on smithi100 2024-09-17T01:07:24.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 118132 8.99 MB/sec execute 203 sec latency 86.934 ms 2024-09-17T01:07:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: pgmap v31: 97 pgs: 97 active+clean; 345 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 396 op/s 2024-09-17T01:07:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T01:07:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:07:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:24 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.6 on smithi100 2024-09-17T01:07:25.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 118551 8.98 MB/sec execute 204 sec latency 65.277 ms 2024-09-17T01:07:25.672 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:25.672 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:25.672 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:25 smithi033 ceph-mon[65430]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T01:07:25.672 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:07:25.672 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:25.672 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:25 smithi033 ceph-mon[65430]: Reconfiguring daemon osd.7 on smithi100 2024-09-17T01:07:25.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:25.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:25.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:25 smithi100 ceph-mon[51701]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T01:07:25.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:07:25.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:25.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:25 smithi100 ceph-mon[51701]: Reconfiguring daemon osd.7 on smithi100 2024-09-17T01:07:26.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 118832 8.97 MB/sec execute 205 sec latency 49.420 ms 2024-09-17T01:07:26.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:26 smithi033 ceph-mon[65430]: pgmap v32: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.0 MiB/s wr, 639 op/s 2024-09-17T01:07:26.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:26.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:26.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:07:26.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:26.782 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:26 smithi100 ceph-mon[51701]: pgmap v32: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 7.0 MiB/s wr, 639 op/s 2024-09-17T01:07:26.783 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:26.783 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:26.783 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:07:26.783 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:27.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 119267 8.99 MB/sec execute 206 sec latency 46.993 ms 2024-09-17T01:07:27.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:27 smithi033 ceph-mon[65430]: Reconfiguring mds.foofs.smithi100.ssfebu (monmap changed)... 2024-09-17T01:07:27.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:27 smithi033 ceph-mon[65430]: Reconfiguring daemon mds.foofs.smithi100.ssfebu on smithi100 2024-09-17T01:07:27.721 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:27.931 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:27 smithi100 ceph-mon[51701]: Reconfiguring mds.foofs.smithi100.ssfebu (monmap changed)... 2024-09-17T01:07:27.931 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:27 smithi100 ceph-mon[51701]: Reconfiguring daemon mds.foofs.smithi100.ssfebu on smithi100 2024-09-17T01:07:27.931 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:28.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 119583 8.96 MB/sec execute 207 sec latency 71.732 ms 2024-09-17T01:07:28.745 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: pgmap v33: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 446 op/s 2024-09-17T01:07:28.745 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:28.746 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:28.746 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:07:28.747 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:28.747 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:28.747 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:28.747 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi033"}]: dispatch 2024-09-17T01:07:28.748 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi033"}]': finished 2024-09-17T01:07:28.748 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi100"}]: dispatch 2024-09-17T01:07:28.748 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi100"}]': finished 2024-09-17T01:07:28.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: pgmap v33: 97 pgs: 97 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 446 op/s 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi033"}]: dispatch 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi033"}]': finished 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi100"}]: dispatch 2024-09-17T01:07:28.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi100"}]': finished 2024-09-17T01:07:29.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 119934 8.94 MB/sec execute 208 sec latency 55.864 ms 2024-09-17T01:07:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:29 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all mon 2024-09-17T01:07:29.772 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:29 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all mon 2024-09-17T01:07:30.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 120204 8.95 MB/sec execute 209 sec latency 53.088 ms 2024-09-17T01:07:30.793 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:30 smithi033 ceph-mon[65430]: pgmap v34: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 646 op/s 2024-09-17T01:07:30.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:30 smithi100 ceph-mon[51701]: pgmap v34: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.3 MiB/s wr, 646 op/s 2024-09-17T01:07:31.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 121025 9.04 MB/sec execute 210 sec latency 54.915 ms 2024-09-17T01:07:32.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:31 smithi033 ceph-mon[65430]: Upgrade: Updating crash.smithi033 (1/2) 2024-09-17T01:07:32.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:31 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:32.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:31 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:32.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:31 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:32.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:31 smithi033 ceph-mon[65430]: Deploying daemon crash.smithi033 on smithi033 2024-09-17T01:07:32.053 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:31 smithi033 ceph-mon[65430]: pgmap v35: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-17T01:07:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:31 smithi100 ceph-mon[51701]: Upgrade: Updating crash.smithi033 (1/2) 2024-09-17T01:07:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:31 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:31 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi033", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:32.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:31 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:32.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:31 smithi100 ceph-mon[51701]: Deploying daemon crash.smithi033 on smithi033 2024-09-17T01:07:32.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:31 smithi100 ceph-mon[51701]: pgmap v35: 97 pgs: 97 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 442 op/s 2024-09-17T01:07:32.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 121411 9.08 MB/sec execute 211 sec latency 89.891 ms 2024-09-17T01:07:33.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 121658 9.06 MB/sec execute 212 sec latency 54.080 ms 2024-09-17T01:07:34.100 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:33 smithi033 ceph-mon[65430]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 543 op/s 2024-09-17T01:07:34.304 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:07:34.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 121964 9.04 MB/sec execute 213 sec latency 45.023 ms 2024-09-17T01:07:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:33 smithi100 ceph-mon[51701]: pgmap v36: 97 pgs: 97 active+clean; 343 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.1 MiB/s wr, 543 op/s 2024-09-17T01:07:35.139 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:35 smithi033 ceph-mon[65430]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:35.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 122166 9.01 MB/sec execute 214 sec latency 56.912 ms 2024-09-17T01:07:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:07:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (3m) 58s ago 11m 17.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:07:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (11m) 58s ago 11m 6405k - 17.2.0 e1d6a67b021e cf5fd15a64af 2024-09-17T01:07:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (11m) 44s ago 11m 6409k - 17.2.0 e1d6a67b021e b23cf8b31a57 2024-09-17T01:07:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (3m) 58s ago 11m 44.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:07:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (6m) 58s ago 6m 35.7M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (6m) 44s ago 6m 14.0M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (2m) 58s ago 13m 473M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (93s) 44s ago 11m 438M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (77s) 58s ago 13m 31.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (51s) 44s ago 11m 22.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (5m) 58s ago 5m 267M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (5m) 44s ago 5m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (3m) 58s ago 11m 8216k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (2m) 44s ago 11m 8334k - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (10m) 58s ago 10m 423M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (9m) 58s ago 9m 313M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (9m) 58s ago 9m 249M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (9m) 58s ago 9m 380M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:07:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (8m) 44s ago 8m 384M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:07:35.343 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (8m) 44s ago 8m 422M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:07:35.343 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (7m) 44s ago 7m 349M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:07:35.343 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (7m) 44s ago 7m 495M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:07:35.343 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (100s) 58s ago 10m 51.3M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:07:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:35 smithi100 ceph-mon[51701]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:36.022 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:07:36.022 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:07:36.022 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:07:36.022 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:36.022 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T01:07:36.023 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:07:36.024 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:07:36.058 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:36 smithi033 ceph-mon[65430]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:36.058 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:36 smithi033 ceph-mon[65430]: pgmap v37: 97 pgs: 97 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.4 MiB/s wr, 686 op/s 2024-09-17T01:07:36.058 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:36 smithi033 ceph-mon[65430]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:36.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 122353 9.00 MB/sec execute 215 sec latency 49.923 ms 2024-09-17T01:07:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:36 smithi100 ceph-mon[51701]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:36 smithi100 ceph-mon[51701]: pgmap v37: 97 pgs: 97 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.4 MiB/s wr, 686 op/s 2024-09-17T01:07:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:36 smithi100 ceph-mon[51701]: from='client.34184 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:36.548 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:07:36.548 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading crash daemons", 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:07:36.549 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:07:37.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:37 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1969339872' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:37.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:37 smithi033 ceph-mon[65430]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:37.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:37.082 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:37 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:37.232 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:07:37.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 122629 8.98 MB/sec execute 216 sec latency 57.219 ms 2024-09-17T01:07:37.410 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:37 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1969339872' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:37.410 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:37 smithi100 ceph-mon[51701]: from='client.34192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:07:37.410 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:37 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:37.410 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:37 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:38.089 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:38 smithi033 ceph-mon[65430]: pgmap v38: 97 pgs: 97 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-17T01:07:38.089 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:38 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2584445296' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:07:38.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 122936 8.98 MB/sec execute 217 sec latency 52.497 ms 2024-09-17T01:07:38.337 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:38 smithi100 ceph-mon[51701]: pgmap v38: 97 pgs: 97 active+clean; 350 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.5 MiB/s wr, 443 op/s 2024-09-17T01:07:38.337 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:38 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2584445296' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:07:39.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 123302 8.95 MB/sec execute 218 sec latency 82.676 ms 2024-09-17T01:07:39.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:39 smithi033 ceph-mon[65430]: Upgrade: Updating crash.smithi100 (2/2) 2024-09-17T01:07:39.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:39.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:39.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:39.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:39 smithi033 ceph-mon[65430]: Deploying daemon crash.smithi100 on smithi100 2024-09-17T01:07:39.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:39 smithi100 ceph-mon[51701]: Upgrade: Updating crash.smithi100 (2/2) 2024-09-17T01:07:39.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:39 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:39.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:39 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi100", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T01:07:39.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:39 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:39.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:39 smithi100 ceph-mon[51701]: Deploying daemon crash.smithi100 on smithi100 2024-09-17T01:07:40.312 INFO:teuthology.orchestra.run.smithi033.stdout: 5 123630 8.95 MB/sec execute 219 sec latency 31.845 ms 2024-09-17T01:07:40.381 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:40 smithi033 ceph-mon[65430]: pgmap v39: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 612 op/s 2024-09-17T01:07:40.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:40 smithi100 ceph-mon[51701]: pgmap v39: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 5.9 MiB/s wr, 612 op/s 2024-09-17T01:07:41.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 124004 8.98 MB/sec execute 220 sec latency 49.621 ms 2024-09-17T01:07:42.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 124376 9.00 MB/sec execute 221 sec latency 51.844 ms 2024-09-17T01:07:42.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:42 smithi033 ceph-mon[65430]: pgmap v40: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 412 op/s 2024-09-17T01:07:42.421 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:42.644 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:42 smithi100 ceph-mon[51701]: pgmap v40: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.2 MiB/s wr, 412 op/s 2024-09-17T01:07:42.644 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:43.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 124694 9.01 MB/sec execute 222 sec latency 53.741 ms 2024-09-17T01:07:43.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:43.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:43.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:07:43.958 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:43 smithi033 ceph-mon[65430]: pgmap v41: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 416 op/s 2024-09-17T01:07:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:07:44.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:43 smithi100 ceph-mon[51701]: pgmap v41: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 416 op/s 2024-09-17T01:07:44.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 124985 9.03 MB/sec execute 223 sec latency 40.366 ms 2024-09-17T01:07:45.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 125235 9.00 MB/sec execute 224 sec latency 43.035 ms 2024-09-17T01:07:46.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:46 smithi033 ceph-mon[65430]: pgmap v42: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 511 op/s 2024-09-17T01:07:46.288 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:46 smithi100 ceph-mon[51701]: pgmap v42: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 511 op/s 2024-09-17T01:07:46.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 125519 8.98 MB/sec execute 225 sec latency 59.481 ms 2024-09-17T01:07:47.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 125725 8.96 MB/sec execute 226 sec latency 51.988 ms 2024-09-17T01:07:47.777 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.777 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.777 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.777 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:47.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:48.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 125920 8.95 MB/sec execute 227 sec latency 60.122 ms 2024-09-17T01:07:48.818 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:48 smithi033 ceph-mon[65430]: pgmap v43: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 369 op/s 2024-09-17T01:07:48.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:48 smithi100 ceph-mon[51701]: pgmap v43: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 369 op/s 2024-09-17T01:07:49.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 126254 8.94 MB/sec execute 228 sec latency 75.904 ms 2024-09-17T01:07:50.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 126516 8.93 MB/sec execute 229 sec latency 72.565 ms 2024-09-17T01:07:50.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:50 smithi033 ceph-mon[65430]: pgmap v44: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 537 op/s 2024-09-17T01:07:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:50 smithi100 ceph-mon[51701]: pgmap v44: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.3 MiB/s wr, 537 op/s 2024-09-17T01:07:51.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 126854 8.90 MB/sec execute 230 sec latency 54.820 ms 2024-09-17T01:07:52.158 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: pgmap v45: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 368 op/s 2024-09-17T01:07:52.158 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.158 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.158 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.158 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.158 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.159 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 127205 8.91 MB/sec execute 231 sec latency 35.465 ms 2024-09-17T01:07:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: pgmap v45: 97 pgs: 97 active+clean; 346 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 368 op/s 2024-09-17T01:07:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:52.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:53.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 127595 8.94 MB/sec execute 232 sec latency 58.949 ms 2024-09-17T01:07:54.245 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:54 smithi033 ceph-mon[65430]: pgmap v46: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 448 op/s 2024-09-17T01:07:54.313 INFO:teuthology.orchestra.run.smithi033.stdout: 5 127932 8.94 MB/sec execute 233 sec latency 54.486 ms 2024-09-17T01:07:54.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:54 smithi100 ceph-mon[51701]: pgmap v46: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 448 op/s 2024-09-17T01:07:55.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 128162 8.93 MB/sec execute 234 sec latency 71.246 ms 2024-09-17T01:07:56.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:56 smithi033 ceph-mon[65430]: pgmap v47: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 557 op/s 2024-09-17T01:07:56.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 128547 8.97 MB/sec execute 235 sec latency 41.506 ms 2024-09-17T01:07:56.415 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:56 smithi100 ceph-mon[51701]: pgmap v47: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.5 MiB/s wr, 557 op/s 2024-09-17T01:07:57.310 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:57.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 128722 8.94 MB/sec execute 236 sec latency 126.557 ms 2024-09-17T01:07:57.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:07:58.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 128892 8.91 MB/sec execute 237 sec latency 62.452 ms 2024-09-17T01:07:58.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: pgmap v48: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.3 MiB/s wr, 361 op/s 2024-09-17T01:07:58.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all crash 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi033"}]: dispatch 2024-09-17T01:07:58.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi033"}]': finished 2024-09-17T01:07:58.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi100"}]: dispatch 2024-09-17T01:07:58.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi100"}]': finished 2024-09-17T01:07:58.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:07:58.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: pgmap v48: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.3 MiB/s wr, 361 op/s 2024-09-17T01:07:58.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all crash 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi033"}]: dispatch 2024-09-17T01:07:58.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi033"}]': finished 2024-09-17T01:07:58.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi100"}]: dispatch 2024-09-17T01:07:58.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi100"}]': finished 2024-09-17T01:07:58.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:07:59.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 129153 8.90 MB/sec execute 238 sec latency 58.533 ms 2024-09-17T01:07:59.655 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:59 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:07:59.655 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:07:59 smithi033 ceph-mon[65430]: Upgrade: osd.0 is safe to restart 2024-09-17T01:07:59.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:59 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T01:08:00.006 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:07:59 smithi100 ceph-mon[51701]: Upgrade: osd.0 is safe to restart 2024-09-17T01:08:00.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 129362 8.89 MB/sec execute 239 sec latency 53.953 ms 2024-09-17T01:08:00.679 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:00 smithi033 ceph-mon[65430]: pgmap v49: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 525 op/s 2024-09-17T01:08:00.679 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:00 smithi033 ceph-mon[65430]: mgrmap e42: smithi033.illmog(active, since 92s), standbys: smithi100.qrwkie 2024-09-17T01:08:00.679 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:00 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:00.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:00 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:08:00.680 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:00 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:08:00.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:00 smithi100 ceph-mon[51701]: pgmap v49: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.4 MiB/s wr, 525 op/s 2024-09-17T01:08:00.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:00 smithi100 ceph-mon[51701]: mgrmap e42: smithi033.illmog(active, since 92s), standbys: smithi100.qrwkie 2024-09-17T01:08:00.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:00 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:00.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:00 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T01:08:00.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:00 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:08:01.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 129571 8.87 MB/sec execute 240 sec latency 63.578 ms 2024-09-17T01:08:01.689 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:01 smithi033 ceph-mon[65430]: Upgrade: Updating osd.0 2024-09-17T01:08:01.689 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:01 smithi033 ceph-mon[65430]: Deploying daemon osd.0 on smithi033 2024-09-17T01:08:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:01 smithi100 ceph-mon[51701]: Upgrade: Updating osd.0 2024-09-17T01:08:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:01 smithi100 ceph-mon[51701]: Deploying daemon osd.0 on smithi033 2024-09-17T01:08:02.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 129937 8.87 MB/sec execute 241 sec latency 43.332 ms 2024-09-17T01:08:02.726 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:02 smithi033 ceph-mon[65430]: pgmap v50: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 356 op/s 2024-09-17T01:08:02.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:02 smithi100 ceph-mon[51701]: pgmap v50: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 356 op/s 2024-09-17T01:08:03.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 130393 8.85 MB/sec execute 242 sec latency 57.890 ms 2024-09-17T01:08:04.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 130732 8.86 MB/sec execute 243 sec latency 54.573 ms 2024-09-17T01:08:04.821 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:04 smithi033 ceph-mon[65430]: pgmap v51: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 360 op/s 2024-09-17T01:08:04.822 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:04 smithi033 systemd[1]: Stopping Ceph osd.0 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:08:04.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:04 smithi100 ceph-mon[51701]: pgmap v51: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.8 MiB/s wr, 360 op/s 2024-09-17T01:08:05.102 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:05 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[34351]: 2024-09-17T01:08:05.046+0000 7f015274d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:08:05.102 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:05 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[34351]: 2024-09-17T01:08:05.046+0000 7f015274d700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-17T01:08:05.102 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:05 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[34351]: 2024-09-17T01:08:05.046+0000 7f015274d700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:08:05.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 131041 8.86 MB/sec execute 244 sec latency 52.877 ms 2024-09-17T01:08:05.635 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:05 smithi033 ceph-mon[65430]: osd.0 marked itself down and dead 2024-09-17T01:08:05.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:05 smithi100 ceph-mon[51701]: osd.0 marked itself down and dead 2024-09-17T01:08:06.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 131264 8.87 MB/sec execute 245 sec latency 690.171 ms 2024-09-17T01:08:06.333 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:06 smithi033 podman[70929]: 2024-09-17 01:08:06.319982026 +0000 UTC m=+1.410510106 container died 16e59b9be15e6d01a574fb4db2b0df5c641e7fc4047000989fdb7dcb619fa895 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, distribution-scope=public, vendor=Red Hat, Inc.) 2024-09-17T01:08:06.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:06 smithi033 ceph-mon[65430]: pgmap v52: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-17T01:08:06.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:06 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:08:06.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:06 smithi033 ceph-mon[65430]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T01:08:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:06 smithi100 ceph-mon[51701]: pgmap v52: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.7 MiB/s wr, 459 op/s 2024-09-17T01:08:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:06 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:08:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:06 smithi100 ceph-mon[51701]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T01:08:07.121 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:07 smithi033 podman[70929]: 2024-09-17 01:08:07.100351829 +0000 UTC m=+2.190879910 container cleanup 16e59b9be15e6d01a574fb4db2b0df5c641e7fc4047000989fdb7dcb619fa895 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, io.buildah.version=1.19.8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, version=8, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc.) 2024-09-17T01:08:07.121 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:07 smithi033 bash[70929]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0 2024-09-17T01:08:07.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 131613 8.87 MB/sec execute 246 sec latency 1020.712 ms 2024-09-17T01:08:07.721 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:08:07.898 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:07 smithi033 ceph-mon[65430]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T01:08:07.898 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:07 smithi033 podman[70944]: 2024-09-17 01:08:07.687129942 +0000 UTC m=+1.366251501 container remove 16e59b9be15e6d01a574fb4db2b0df5c641e7fc4047000989fdb7dcb619fa895 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , ceph=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, io.openshift.expose-services=, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, architecture=x86_64, release=754, build-date=2022-05-03T08:36:31.336870) 2024-09-17T01:08:07.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:07 smithi100 ceph-mon[51701]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T01:08:08.152 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:08 smithi033 podman[71076]: 2024-09-17 01:08:08.037361171 +0000 UTC m=+0.025145686 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:08:08.152 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:08 smithi033 podman[71076]: 2024-09-17 01:08:08.154991191 +0000 UTC m=+0.142775688 container create 5c04c647fd47babc4459165ed063c6151fd56a365791932fe4b79f5810fcb789 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:08:08.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 131873 8.89 MB/sec execute 247 sec latency 71.954 ms 2024-09-17T01:08:08.654 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:08 smithi033 ceph-mon[65430]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.0 MiB/s wr, 273 op/s 2024-09-17T01:08:08.654 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:08 smithi033 podman[71076]: 2024-09-17 01:08:08.529641535 +0000 UTC m=+0.517426047 container init 5c04c647fd47babc4459165ed063c6151fd56a365791932fe4b79f5810fcb789 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T01:08:08.654 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:08 smithi033 podman[71076]: 2024-09-17 01:08:08.534984289 +0000 UTC m=+0.522768800 container start 5c04c647fd47babc4459165ed063c6151fd56a365791932fe4b79f5810fcb789 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-09-17T01:08:08.667 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:08:08.667 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (3m) 17s ago 12m 26.8M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:08:08.667 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (32s) 17s ago 12m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (26s) 22s ago 11m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (3m) 17s ago 12m 44.7M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (6m) 17s ago 6m 36.2M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (6m) 22s ago 6m 14.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (3m) 17s ago 13m 494M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (2m) 22s ago 11m 440M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (110s) 17s ago 13m 43.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (84s) 22s ago 11m 36.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (6m) 17s ago 6m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (6m) 22s ago 6m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (3m) 17s ago 12m 8610k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (3m) 22s ago 11m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (11m) 17s ago 11m 480M 1937M 17.2.0 e1d6a67b021e 16e59b9be15e 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (10m) 17s ago 10m 403M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:08:08.668 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (10m) 17s ago 10m 289M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:08:08.669 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (9m) 17s ago 9m 427M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:08:08.669 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (9m) 22s ago 9m 449M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:08:08.669 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (8m) 22s ago 8m 491M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:08:08.669 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (8m) 22s ago 8m 409M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:08:08.669 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (8m) 22s ago 8m 572M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:08:08.669 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (2m) 17s ago 11m 57.5M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:08:08.929 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:08 smithi033 podman[71076]: 2024-09-17 01:08:08.659741162 +0000 UTC m=+0.647525667 container attach 5c04c647fd47babc4459165ed063c6151fd56a365791932fe4b79f5810fcb789 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T01:08:08.929 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:08 smithi033 podman[71076]: 2024-09-17 01:08:08.824792454 +0000 UTC m=+0.812576963 container died 5c04c647fd47babc4459165ed063c6151fd56a365791932fe4b79f5810fcb789 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97) 2024-09-17T01:08:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:08 smithi100 ceph-mon[51701]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.0 MiB/s wr, 273 op/s 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:08:09.166 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:08:09.167 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:08:09.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 131998 8.87 MB/sec execute 248 sec latency 104.423 ms 2024-09-17T01:08:09.577 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:08:09.577 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:08:09.577 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "6/23 daemons upgraded", 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:08:09.578 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:08:09.698 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:09 smithi033 ceph-mon[65430]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:09.698 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:09 smithi033 ceph-mon[65430]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:09.698 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:09 smithi033 ceph-mon[65430]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:09.698 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:09 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/844921159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:09.698 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:09 smithi033 podman[71143]: 2024-09-17 01:08:09.574232166 +0000 UTC m=+0.740790547 container remove 5c04c647fd47babc4459165ed063c6151fd56a365791932fe4b79f5810fcb789 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:08:09.699 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:09 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T01:08:09.699 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:09 smithi033 systemd[1]: Stopped Ceph osd.0 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:08:09.699 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:09 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 8.414s CPU time. 2024-09-17T01:08:09.950 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:09 smithi033 systemd[1]: Starting Ceph osd.0 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:08:09.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:09 smithi100 ceph-mon[51701]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:09.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:09 smithi100 ceph-mon[51701]: from='client.34204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:09.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:09 smithi100 ceph-mon[51701]: from='client.34208 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:09.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:09 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/844921159' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:10.135 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 228/1644 objects degraded (13.869%), 29 pgs degraded 2024-09-17T01:08:10.135 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T01:08:10.136 INFO:teuthology.orchestra.run.smithi033.stdout: osd.0 (root=default,host=smithi033) is down 2024-09-17T01:08:10.136 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 228/1644 objects degraded (13.869%), 29 pgs degraded 2024-09-17T01:08:10.136 INFO:teuthology.orchestra.run.smithi033.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-17T01:08:10.136 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-17T01:08:10.136 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-17T01:08:10.136 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.f is active+undersized+degraded, acting [4,2] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-17T01:08:10.137 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-17T01:08:10.138 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-17T01:08:10.206 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 podman[71283]: 2024-09-17 01:08:10.082335162 +0000 UTC m=+0.020540178 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:08:10.206 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 podman[71283]: 2024-09-17 01:08:10.208791996 +0000 UTC m=+0.146996996 container create e98f38ebe3f2fb0166843371bcbe49675074cc690e997cacc66544219e9794ec (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T01:08:10.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 132153 8.85 MB/sec execute 249 sec latency 75.791 ms 2024-09-17T01:08:10.707 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:10 smithi033 ceph-mon[65430]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 535 op/s; 228/1644 objects degraded (13.869%) 2024-09-17T01:08:10.707 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:10 smithi033 ceph-mon[65430]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:10.707 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:10 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 228/1644 objects degraded (13.869%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:10.707 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:10 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2716515592' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:08:10.708 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 podman[71283]: 2024-09-17 01:08:10.621979218 +0000 UTC m=+0.560184219 container init e98f38ebe3f2fb0166843371bcbe49675074cc690e997cacc66544219e9794ec (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:08:10.708 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 podman[71283]: 2024-09-17 01:08:10.63119817 +0000 UTC m=+0.569403178 container start e98f38ebe3f2fb0166843371bcbe49675074cc690e997cacc66544219e9794ec (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS) 2024-09-17T01:08:10.923 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 podman[71283]: 2024-09-17 01:08:10.769021091 +0000 UTC m=+0.707226097 container attach e98f38ebe3f2fb0166843371bcbe49675074cc690e997cacc66544219e9794ec (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T01:08:10.923 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:10.923 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 bash[71283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:10.923 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:10.923 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:10 smithi033 bash[71283]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:10 smithi100 ceph-mon[51701]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.2 MiB/s wr, 535 op/s; 228/1644 objects degraded (13.869%) 2024-09-17T01:08:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:10 smithi100 ceph-mon[51701]: from='client.34216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:10 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 228/1644 objects degraded (13.869%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:10.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:10 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2716515592' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:08:11.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 132428 8.83 MB/sec execute 250 sec latency 49.537 ms 2024-09-17T01:08:12.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 132710 8.82 MB/sec execute 251 sec latency 59.294 ms 2024-09-17T01:08:12.539 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:08:12.539 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:08:12.539 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:08:12.540 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:08:12.790 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-mon[65430]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 529 op/s; 228/1644 objects degraded (13.869%) 2024-09-17T01:08:12.790 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:12.790 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:12.790 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate[71299]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 bash[71283]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T01:08:12.791 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:12 smithi033 podman[71283]: 2024-09-17 01:08:12.767932294 +0000 UTC m=+2.706137306 container died e98f38ebe3f2fb0166843371bcbe49675074cc690e997cacc66544219e9794ec (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-17T01:08:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:12 smithi100 ceph-mon[51701]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 5.1 MiB/s wr, 529 op/s; 228/1644 objects degraded (13.869%) 2024-09-17T01:08:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:12.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:13.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 132953 8.81 MB/sec execute 252 sec latency 55.499 ms 2024-09-17T01:08:13.821 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:13 smithi033 ceph-mon[65430]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.2 MiB/s wr, 267 op/s; 229/1656 objects degraded (13.829%) 2024-09-17T01:08:13.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:13 smithi100 ceph-mon[51701]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.2 MiB/s wr, 267 op/s; 229/1656 objects degraded (13.829%) 2024-09-17T01:08:14.073 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:13 smithi033 podman[71448]: 2024-09-17 01:08:13.966356429 +0000 UTC m=+1.190330612 container remove e98f38ebe3f2fb0166843371bcbe49675074cc690e997cacc66544219e9794ec (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T01:08:14.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 133294 8.81 MB/sec execute 253 sec latency 40.460 ms 2024-09-17T01:08:14.323 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 podman[71502]: 2024-09-17 01:08:14.121099718 +0000 UTC m=+0.027972041 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:08:14.323 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 podman[71502]: 2024-09-17 01:08:14.256722393 +0000 UTC m=+0.163594702 container create fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-17T01:08:14.579 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 podman[71502]: 2024-09-17 01:08:14.568716702 +0000 UTC m=+0.475589019 container init fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T01:08:14.645 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 podman[71502]: 2024-09-17 01:08:14.581798992 +0000 UTC m=+0.488671306 container start fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T01:08:14.645 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 ceph-osd[71521]: -- 172.21.15.33:0/1417541998 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5622f46041e0 con 0x5622f39ca000 2024-09-17T01:08:14.829 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 bash[71502]: fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 2024-09-17T01:08:14.829 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:14 smithi033 systemd[1]: Started Ceph osd.0 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:08:15.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 133796 8.80 MB/sec execute 254 sec latency 42.134 ms 2024-09-17T01:08:15.916 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:15.916 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:15.917 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:15 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:08:15.917 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:15 smithi033 ceph-mon[65430]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 414 op/s; 230/1686 objects degraded (13.642%) 2024-09-17T01:08:16.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:16.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:16.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:15 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:08:16.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:15 smithi100 ceph-mon[51701]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.7 MiB/s wr, 414 op/s; 230/1686 objects degraded (13.642%) 2024-09-17T01:08:16.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 134306 8.82 MB/sec execute 255 sec latency 41.321 ms 2024-09-17T01:08:16.420 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:16 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:08:16.202+0000 7fb3ab58e640 -1 Falling back to public interface 2024-09-17T01:08:16.955 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:16 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 230/1686 objects degraded (13.642%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:17.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:16 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 230/1686 objects degraded (13.642%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:17.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 134831 8.86 MB/sec execute 256 sec latency 58.162 ms 2024-09-17T01:08:17.992 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:17 smithi033 ceph-mon[65430]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 374 op/s; 230/1686 objects degraded (13.642%) 2024-09-17T01:08:18.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:17 smithi100 ceph-mon[51701]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.2 MiB/s wr, 374 op/s; 230/1686 objects degraded (13.642%) 2024-09-17T01:08:18.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 135201 8.88 MB/sec execute 257 sec latency 49.480 ms 2024-09-17T01:08:19.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 135424 8.88 MB/sec execute 258 sec latency 69.791 ms 2024-09-17T01:08:20.035 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:20 smithi033 ceph-mon[65430]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 579 op/s; 214/1710 objects degraded (12.515%) 2024-09-17T01:08:20.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 135592 8.88 MB/sec execute 259 sec latency 56.644 ms 2024-09-17T01:08:20.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:20 smithi100 ceph-mon[51701]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 579 op/s; 214/1710 objects degraded (12.515%) 2024-09-17T01:08:21.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 135982 8.86 MB/sec execute 260 sec latency 65.422 ms 2024-09-17T01:08:22.069 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:22 smithi033 ceph-mon[65430]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 214/1710 objects degraded (12.515%) 2024-09-17T01:08:22.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 136278 8.85 MB/sec execute 261 sec latency 56.612 ms 2024-09-17T01:08:22.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:22 smithi100 ceph-mon[51701]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 214/1710 objects degraded (12.515%) 2024-09-17T01:08:23.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:23 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 215/1701 objects degraded (12.640%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:23.314 INFO:teuthology.orchestra.run.smithi033.stdout: 5 136530 8.85 MB/sec execute 262 sec latency 54.803 ms 2024-09-17T01:08:23.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:23 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 215/1701 objects degraded (12.640%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:23.877 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:23 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:08:23.823+0000 7fb3ab58e640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T01:08:24.141 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:24 smithi033 ceph-mon[65430]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 459 op/s; 215/1701 objects degraded (12.640%) 2024-09-17T01:08:24.142 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:24.142 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:24.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 136811 8.84 MB/sec execute 263 sec latency 56.757 ms 2024-09-17T01:08:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:24 smithi100 ceph-mon[51701]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 459 op/s; 215/1701 objects degraded (12.640%) 2024-09-17T01:08:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:24.897 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:24 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:08:24.753+0000 7fb3ab58e640 -1 osd.0 72 log_to_monitors true 2024-09-17T01:08:25.164 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:25 smithi033 ceph-mon[65430]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:08:25.164 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:25.164 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:25.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 137283 8.83 MB/sec execute 264 sec latency 111.410 ms 2024-09-17T01:08:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:25 smithi100 ceph-mon[51701]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T01:08:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:25.666 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:08:25 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:08:25.637+0000 7fb39f331640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:08:26.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:26 smithi033 ceph-mon[65430]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 583 op/s; 230/1794 objects degraded (12.821%) 2024-09-17T01:08:26.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:26 smithi033 ceph-mon[65430]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:08:26.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:26 smithi033 ceph-mon[65430]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T01:08:26.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:26 smithi033 ceph-mon[65430]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:08:26.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:26 smithi033 ceph-mon[65430]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' 2024-09-17T01:08:26.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 137577 8.81 MB/sec execute 265 sec latency 136.610 ms 2024-09-17T01:08:26.377 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:26 smithi100 ceph-mon[51701]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.9 MiB/s wr, 583 op/s; 230/1794 objects degraded (12.821%) 2024-09-17T01:08:26.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:26 smithi100 ceph-mon[51701]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T01:08:26.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:26 smithi100 ceph-mon[51701]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T01:08:26.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:26 smithi100 ceph-mon[51701]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:08:26.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:26 smithi100 ceph-mon[51701]: from='osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369]' entity='osd.0' 2024-09-17T01:08:27.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 137681 8.79 MB/sec execute 266 sec latency 708.995 ms 2024-09-17T01:08:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:27 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:08:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:27 smithi033 ceph-mon[65430]: osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369] boot 2024-09-17T01:08:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:27 smithi033 ceph-mon[65430]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T01:08:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:08:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:27.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:27.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:27 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:08:27.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:27 smithi100 ceph-mon[51701]: osd.0 [v2:172.21.15.33:6802/1792169369,v1:172.21.15.33:6803/1792169369] boot 2024-09-17T01:08:27.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:27 smithi100 ceph-mon[51701]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T01:08:27.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T01:08:27.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:27.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:28.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 137681 8.76 MB/sec execute 267 sec latency 1709.042 ms 2024-09-17T01:08:28.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:28 smithi033 ceph-mon[65430]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.0 MiB/s wr, 272 op/s; 230/1794 objects degraded (12.821%) 2024-09-17T01:08:28.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:28 smithi033 ceph-mon[65430]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T01:08:28.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:28 smithi100 ceph-mon[51701]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.0 MiB/s wr, 272 op/s; 230/1794 objects degraded (12.821%) 2024-09-17T01:08:28.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:28 smithi100 ceph-mon[51701]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T01:08:29.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 137787 8.74 MB/sec execute 268 sec latency 2282.937 ms 2024-09-17T01:08:29.657 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:29 smithi033 ceph-mon[65430]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T01:08:29.657 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:29 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 230/1794 objects degraded (12.821%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:29.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:29 smithi100 ceph-mon[51701]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T01:08:29.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:29 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 230/1794 objects degraded (12.821%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:30.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 138033 8.74 MB/sec execute 269 sec latency 62.981 ms 2024-09-17T01:08:30.683 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: pgmap v70: 97 pgs: 10 remapped+peering, 5 active+undersized, 5 active+undersized+degraded, 77 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.3 MiB/s wr, 445 op/s; 55/1548 objects degraded (3.553%); 29 KiB/s, 6 objects/s recovering 2024-09-17T01:08:30.683 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T01:08:30.683 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.683 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:30.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:30.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: pgmap v70: 97 pgs: 10 remapped+peering, 5 active+undersized, 5 active+undersized+degraded, 77 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.3 MiB/s wr, 445 op/s; 55/1548 objects degraded (3.553%); 29 KiB/s, 6 objects/s recovering 2024-09-17T01:08:30.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T01:08:30.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:30.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:31.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 138307 8.77 MB/sec execute 270 sec latency 476.856 ms 2024-09-17T01:08:31.734 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:31 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:31.734 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:31 smithi033 ceph-mon[65430]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-17T01:08:31.734 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:31 smithi033 ceph-mon[65430]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T01:08:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:31 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:31 smithi100 ceph-mon[51701]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-17T01:08:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:31 smithi100 ceph-mon[51701]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T01:08:32.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 138613 8.76 MB/sec execute 271 sec latency 811.148 ms 2024-09-17T01:08:32.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:32 smithi033 ceph-mon[65430]: pgmap v73: 97 pgs: 10 remapped+peering, 87 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.4 MiB/s wr, 464 op/s; 29 KiB/s, 22 objects/s recovering 2024-09-17T01:08:32.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:32 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/1548 objects degraded (3.553%), 5 pgs degraded) 2024-09-17T01:08:32.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:32 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:08:32.757 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:32 smithi033 ceph-mon[65430]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T01:08:32.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:32 smithi100 ceph-mon[51701]: pgmap v73: 97 pgs: 10 remapped+peering, 87 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 3.4 MiB/s wr, 464 op/s; 29 KiB/s, 22 objects/s recovering 2024-09-17T01:08:32.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:32 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/1548 objects degraded (3.553%), 5 pgs degraded) 2024-09-17T01:08:32.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:32 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:08:32.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:32 smithi100 ceph-mon[51701]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T01:08:33.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 138928 8.77 MB/sec execute 272 sec latency 168.754 ms 2024-09-17T01:08:33.774 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:33 smithi033 ceph-mon[65430]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T01:08:33.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:33 smithi100 ceph-mon[51701]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T01:08:34.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 139125 8.76 MB/sec execute 273 sec latency 460.111 ms 2024-09-17T01:08:34.780 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:34 smithi033 ceph-mon[65430]: pgmap v76: 97 pgs: 9 remapped+peering, 88 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s wr, 250 op/s; 0 B/s, 22 objects/s recovering 2024-09-17T01:08:34.780 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:34 smithi033 ceph-mon[65430]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T01:08:34.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:34 smithi100 ceph-mon[51701]: pgmap v76: 97 pgs: 9 remapped+peering, 88 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s wr, 250 op/s; 0 B/s, 22 objects/s recovering 2024-09-17T01:08:34.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:34 smithi100 ceph-mon[51701]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T01:08:35.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 139292 8.73 MB/sec execute 274 sec latency 490.834 ms 2024-09-17T01:08:35.818 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:35 smithi033 ceph-mon[65430]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T01:08:35.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:35 smithi100 ceph-mon[51701]: osdmap e84: 8 total, 8 up, 8 in 2024-09-17T01:08:36.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 139697 8.73 MB/sec execute 275 sec latency 858.135 ms 2024-09-17T01:08:36.829 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:36 smithi033 ceph-mon[65430]: pgmap v79: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 92 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 348 op/s; 22/1620 objects degraded (1.358%); 2.0 MiB/s, 28 objects/s recovering 2024-09-17T01:08:36.829 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:36 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 22/1620 objects degraded (1.358%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:36.829 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:36 smithi033 ceph-mon[65430]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T01:08:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:36 smithi100 ceph-mon[51701]: pgmap v79: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 92 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 4.1 MiB/s wr, 348 op/s; 22/1620 objects degraded (1.358%); 2.0 MiB/s, 28 objects/s recovering 2024-09-17T01:08:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:36 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 22/1620 objects degraded (1.358%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:36 smithi100 ceph-mon[51701]: osdmap e85: 8 total, 8 up, 8 in 2024-09-17T01:08:37.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 140242 8.76 MB/sec execute 276 sec latency 36.604 ms 2024-09-17T01:08:38.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 140627 8.76 MB/sec execute 277 sec latency 44.164 ms 2024-09-17T01:08:38.882 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:38 smithi033 ceph-mon[65430]: pgmap v81: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.7 MiB/s wr, 352 op/s; 21/1644 objects degraded (1.277%); 1.9 MiB/s, 36 objects/s recovering 2024-09-17T01:08:38.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:38 smithi100 ceph-mon[51701]: pgmap v81: 97 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.7 MiB/s wr, 352 op/s; 21/1644 objects degraded (1.277%); 1.9 MiB/s, 36 objects/s recovering 2024-09-17T01:08:39.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 141026 8.74 MB/sec execute 278 sec latency 49.755 ms 2024-09-17T01:08:39.918 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:39 smithi033 ceph-mon[65430]: pgmap v82: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.1 MiB/s wr, 504 op/s; 1.3 MiB/s, 25 objects/s recovering 2024-09-17T01:08:39.918 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:39 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1644 objects degraded (1.277%), 1 pg degraded) 2024-09-17T01:08:39.918 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:39 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:08:39.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:39 smithi100 ceph-mon[51701]: pgmap v82: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.1 MiB/s wr, 504 op/s; 1.3 MiB/s, 25 objects/s recovering 2024-09-17T01:08:39.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:39 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1644 objects degraded (1.277%), 1 pg degraded) 2024-09-17T01:08:39.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:39 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:08:40.315 INFO:teuthology.orchestra.run.smithi033.stdout: 5 141258 8.74 MB/sec execute 279 sec latency 61.962 ms 2024-09-17T01:08:40.605 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:08:40.941 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:40 smithi033 ceph-mon[65430]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:40.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:40 smithi100 ceph-mon[51701]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:41.321 INFO:teuthology.orchestra.run.smithi033.stdout: 5 141597 8.76 MB/sec execute 280 sec latency 60.177 ms 2024-09-17T01:08:41.540 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:08:41.540 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (4m) 17s ago 13m 26.8M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:08:41.540 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (65s) 17s ago 13m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (59s) 54s ago 12m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (4m) 17s ago 12m 46.0M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (7m) 17s ago 7m 36.2M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (7m) 54s ago 7m 14.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (3m) 17s ago 14m 496M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (2m) 54s ago 12m 440M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (2m) 17s ago 14m 46.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (117s) 54s ago 12m 36.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (6m) 17s ago 6m 272M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (6m) 54s ago 6m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (4m) 17s ago 12m 8619k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (3m) 54s ago 12m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:08:41.541 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (27s) 17s ago 11m 14.4M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (11m) 17s ago 11m 420M 1937M 17.2.0 e1d6a67b021e 75ef5813ee0d 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (10m) 17s ago 10m 299M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (10m) 17s ago 10m 454M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (9m) 54s ago 9m 449M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (9m) 54s ago 9m 491M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (9m) 54s ago 9m 409M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (8m) 54s ago 8m 572M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:08:41.542 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (2m) 17s ago 12m 57.8M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:08:41.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:41 smithi033 ceph-mon[65430]: pgmap v83: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 382 op/s; 0 B/s, 9 objects/s recovering 2024-09-17T01:08:41.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:41 smithi033 ceph-mon[65430]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:41.957 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:41 smithi033 ceph-mon[65430]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:42.081 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:08:42.083 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:08:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:41 smithi100 ceph-mon[51701]: pgmap v83: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 382 op/s; 0 B/s, 9 objects/s recovering 2024-09-17T01:08:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:41 smithi100 ceph-mon[51701]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:41 smithi100 ceph-mon[51701]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:42.321 INFO:teuthology.orchestra.run.smithi033.stdout: 5 141876 8.76 MB/sec execute 281 sec latency 47.430 ms 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:08:42.557 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:08:42.558 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "7/23 daemons upgraded", 2024-09-17T01:08:42.558 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:08:42.558 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:08:42.558 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:08:43.052 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:08:43.264 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:43.264 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:43.264 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:43 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2101895841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:43.264 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:43 smithi033 ceph-mon[65430]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:43.321 INFO:teuthology.orchestra.run.smithi033.stdout: 5 142118 8.76 MB/sec execute 282 sec latency 68.854 ms 2024-09-17T01:08:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:43 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2101895841' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:08:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:43 smithi100 ceph-mon[51701]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:08:44.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:44 smithi033 ceph-mon[65430]: pgmap v84: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.9 MiB/s wr, 433 op/s; 0 B/s, 7 objects/s recovering 2024-09-17T01:08:44.041 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:44 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2695161470' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:08:44.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 142522 8.78 MB/sec execute 283 sec latency 47.866 ms 2024-09-17T01:08:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:44 smithi100 ceph-mon[51701]: pgmap v84: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.9 MiB/s wr, 433 op/s; 0 B/s, 7 objects/s recovering 2024-09-17T01:08:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:44 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2695161470' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:08:45.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 142887 8.76 MB/sec execute 284 sec latency 39.646 ms 2024-09-17T01:08:46.066 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:46 smithi033 ceph-mon[65430]: pgmap v85: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 481 op/s; 0 B/s, 6 objects/s recovering 2024-09-17T01:08:46.066 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:46 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:46.066 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:46 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:46.066 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:46 smithi033 ceph-mon[65430]: Upgrade: osd.1 is safe to restart 2024-09-17T01:08:46.321 INFO:teuthology.orchestra.run.smithi033.stdout: 5 143159 8.74 MB/sec execute 285 sec latency 43.263 ms 2024-09-17T01:08:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:46 smithi100 ceph-mon[51701]: pgmap v85: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.0 MiB/s wr, 481 op/s; 0 B/s, 6 objects/s recovering 2024-09-17T01:08:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:46 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:46 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T01:08:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:46 smithi100 ceph-mon[51701]: Upgrade: osd.1 is safe to restart 2024-09-17T01:08:47.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 143480 8.75 MB/sec execute 286 sec latency 49.285 ms 2024-09-17T01:08:48.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: pgmap v86: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 427 op/s; 0 B/s, 1 objects/s recovering 2024-09-17T01:08:48.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:48.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: Upgrade: Updating osd.1 2024-09-17T01:08:48.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:48.093 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:08:48.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:08:48.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:48 smithi033 ceph-mon[65430]: Deploying daemon osd.1 on smithi033 2024-09-17T01:08:48.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 143926 8.76 MB/sec execute 287 sec latency 44.619 ms 2024-09-17T01:08:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: pgmap v86: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 427 op/s; 0 B/s, 1 objects/s recovering 2024-09-17T01:08:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: Upgrade: Updating osd.1 2024-09-17T01:08:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T01:08:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:08:48.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:48 smithi100 ceph-mon[51701]: Deploying daemon osd.1 on smithi033 2024-09-17T01:08:49.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 144333 8.76 MB/sec execute 288 sec latency 62.200 ms 2024-09-17T01:08:50.129 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:50 smithi033 ceph-mon[65430]: pgmap v87: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 0 B/s, 1 objects/s recovering 2024-09-17T01:08:50.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 144674 8.75 MB/sec execute 289 sec latency 38.726 ms 2024-09-17T01:08:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:50 smithi100 ceph-mon[51701]: pgmap v87: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.9 MiB/s wr, 606 op/s; 0 B/s, 1 objects/s recovering 2024-09-17T01:08:51.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 144819 8.74 MB/sec execute 290 sec latency 66.820 ms 2024-09-17T01:08:52.300 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:52 smithi033 ceph-mon[65430]: pgmap v88: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 418 op/s 2024-09-17T01:08:52.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 145067 8.74 MB/sec execute 291 sec latency 59.092 ms 2024-09-17T01:08:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:52 smithi100 ceph-mon[51701]: pgmap v88: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.4 MiB/s wr, 418 op/s 2024-09-17T01:08:52.560 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:52 smithi033 systemd[1]: Stopping Ceph osd.1 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:08:52.817 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:52 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[38252]: 2024-09-17T01:08:52.729+0000 7f58a370a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:08:52.817 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:52 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[38252]: 2024-09-17T01:08:52.729+0000 7f58a370a700 -1 osd.1 85 *** Got signal Terminated *** 2024-09-17T01:08:52.817 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:52 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[38252]: 2024-09-17T01:08:52.729+0000 7f58a370a700 -1 osd.1 85 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:08:53.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 145316 8.75 MB/sec execute 292 sec latency 278.830 ms 2024-09-17T01:08:53.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:53 smithi033 ceph-mon[65430]: osd.1 marked itself down and dead 2024-09-17T01:08:53.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:53 smithi100 ceph-mon[51701]: osd.1 marked itself down and dead 2024-09-17T01:08:53.863 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:53 smithi033 podman[73674]: 2024-09-17 01:08:53.706491274 +0000 UTC m=+1.099491781 container died 75ef5813ee0d8f73dbc41cbbc1247f048b63fb5d5371a70bebe895f3b6a7ba9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.openshift.expose-services=, release=754, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8) 2024-09-17T01:08:54.221 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:54 smithi033 ceph-mon[65430]: pgmap v89: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T01:08:54.221 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:54 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:08:54.221 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:54 smithi033 ceph-mon[65430]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T01:08:54.222 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:54 smithi033 podman[73674]: 2024-09-17 01:08:54.087881073 +0000 UTC m=+1.480881580 container cleanup 75ef5813ee0d8f73dbc41cbbc1247f048b63fb5d5371a70bebe895f3b6a7ba9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, io.openshift.expose-services=, version=8, RELEASE=HEAD, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870) 2024-09-17T01:08:54.222 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:54 smithi033 bash[73674]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1 2024-09-17T01:08:54.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 145381 8.72 MB/sec execute 293 sec latency 1049.078 ms 2024-09-17T01:08:54.472 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:54 smithi033 podman[73689]: 2024-09-17 01:08:54.415010225 +0000 UTC m=+0.698811183 container remove 75ef5813ee0d8f73dbc41cbbc1247f048b63fb5d5371a70bebe895f3b6a7ba9b (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, architecture=x86_64, name=centos-stream, RELEASE=HEAD, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, version=8, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_CLEAN=True, vcs-type=git) 2024-09-17T01:08:54.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:54 smithi100 ceph-mon[51701]: pgmap v89: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 402 op/s 2024-09-17T01:08:54.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:54 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:08:54.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:54 smithi100 ceph-mon[51701]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T01:08:54.974 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:54 smithi033 podman[73777]: 2024-09-17 01:08:54.689308864 +0000 UTC m=+0.020744610 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:08:54.974 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:54 smithi033 podman[73777]: 2024-09-17 01:08:54.850069216 +0000 UTC m=+0.181504958 container create f2477909277f4b044294cbbaed7678389906b2148045a15c46eea299d07d668d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T01:08:55.225 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:55 smithi033 ceph-mon[65430]: osdmap e87: 8 total, 7 up, 8 in 2024-09-17T01:08:55.225 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:55 smithi033 podman[73777]: 2024-09-17 01:08:55.220731495 +0000 UTC m=+0.552167258 container init f2477909277f4b044294cbbaed7678389906b2148045a15c46eea299d07d668d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T01:08:55.226 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:55 smithi033 podman[73777]: 2024-09-17 01:08:55.22784948 +0000 UTC m=+0.559285220 container start f2477909277f4b044294cbbaed7678389906b2148045a15c46eea299d07d668d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, ceph=True) 2024-09-17T01:08:55.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 145613 8.71 MB/sec execute 294 sec latency 75.755 ms 2024-09-17T01:08:55.475 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:55 smithi033 podman[73777]: 2024-09-17 01:08:55.358293708 +0000 UTC m=+0.689729454 container attach f2477909277f4b044294cbbaed7678389906b2148045a15c46eea299d07d668d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T01:08:55.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:55 smithi100 ceph-mon[51701]: osdmap e87: 8 total, 7 up, 8 in 2024-09-17T01:08:55.726 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:55 smithi033 podman[73777]: 2024-09-17 01:08:55.551599862 +0000 UTC m=+0.883035598 container died f2477909277f4b044294cbbaed7678389906b2148045a15c46eea299d07d668d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:08:56.233 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:56 smithi033 ceph-mon[65430]: pgmap v92: 97 pgs: 12 stale+active+clean, 85 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 586 op/s 2024-09-17T01:08:56.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 145950 8.74 MB/sec execute 295 sec latency 55.753 ms 2024-09-17T01:08:56.377 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:56 smithi100 ceph-mon[51701]: pgmap v92: 97 pgs: 12 stale+active+clean, 85 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.5 MiB/s wr, 586 op/s 2024-09-17T01:08:56.755 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:56 smithi033 podman[73801]: 2024-09-17 01:08:56.61249575 +0000 UTC m=+1.047999434 container remove f2477909277f4b044294cbbaed7678389906b2148045a15c46eea299d07d668d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T01:08:56.755 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:56 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T01:08:56.755 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:56 smithi033 systemd[1]: Stopped Ceph osd.1 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:08:56.755 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:56 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 19.057s CPU time. 2024-09-17T01:08:57.022 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:56 smithi033 systemd[1]: Starting Ceph osd.1 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:08:57.275 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 podman[73898]: 2024-09-17 01:08:57.131422938 +0000 UTC m=+0.020728415 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:08:57.275 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 podman[73898]: 2024-09-17 01:08:57.238337716 +0000 UTC m=+0.127643170 container create c9972356eb88834f5db71c174b8c6aee88c02106aa19782665ab1627059ee033 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:08:57.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 146166 8.72 MB/sec execute 296 sec latency 58.474 ms 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 podman[73898]: 2024-09-17 01:08:57.566645507 +0000 UTC m=+0.455950969 container init c9972356eb88834f5db71c174b8c6aee88c02106aa19782665ab1627059ee033 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 podman[73898]: 2024-09-17 01:08:57.573092867 +0000 UTC m=+0.462398322 container start c9972356eb88834f5db71c174b8c6aee88c02106aa19782665ab1627059ee033 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 podman[73898]: 2024-09-17 01:08:57.691871521 +0000 UTC m=+0.581176972 container attach c9972356eb88834f5db71c174b8c6aee88c02106aa19782665ab1627059ee033 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 bash[73898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:57.785 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:57 smithi033 bash[73898]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:08:58.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:58 smithi033 ceph-mon[65430]: pgmap v93: 97 pgs: 1 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 83 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 290 op/s; 15/1632 objects degraded (0.919%) 2024-09-17T01:08:58.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:58.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:58.043 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:08:58 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 15/1632 objects degraded (0.919%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:58.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 146317 8.70 MB/sec execute 297 sec latency 86.714 ms 2024-09-17T01:08:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:58 smithi100 ceph-mon[51701]: pgmap v93: 97 pgs: 1 active+undersized, 11 stale+active+clean, 2 active+undersized+degraded, 83 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.1 MiB/s wr, 290 op/s; 15/1632 objects degraded (0.919%) 2024-09-17T01:08:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:08:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:08:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:08:58 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 15/1632 objects degraded (0.919%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T01:08:59.059 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:58 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:08:59.059 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:58 smithi033 bash[73898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:08:59.059 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:58 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:08:59.059 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:58 smithi033 bash[73898]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:08:59.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 146492 8.67 MB/sec execute 298 sec latency 93.111 ms 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 bash[73898]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 bash[73898]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 bash[73898]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T01:08:59.325 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:08:59.326 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 bash[73898]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T01:08:59.326 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate[73914]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T01:08:59.326 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 bash[73898]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T01:08:59.576 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:08:59 smithi033 podman[73898]: 2024-09-17 01:08:59.345430041 +0000 UTC m=+2.234735514 container died c9972356eb88834f5db71c174b8c6aee88c02106aa19782665ab1627059ee033 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:09:00.080 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:00 smithi033 ceph-mon[65430]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 437 op/s; 183/1659 objects degraded (11.031%) 2024-09-17T01:09:00.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 146882 8.68 MB/sec execute 299 sec latency 48.619 ms 2024-09-17T01:09:00.337 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:00 smithi033 podman[74053]: 2024-09-17 01:09:00.165224114 +0000 UTC m=+0.803431233 container remove c9972356eb88834f5db71c174b8c6aee88c02106aa19782665ab1627059ee033 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:09:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:00 smithi100 ceph-mon[51701]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.5 MiB/s wr, 437 op/s; 183/1659 objects degraded (11.031%) 2024-09-17T01:09:00.598 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:00 smithi033 podman[74143]: 2024-09-17 01:09:00.310927321 +0000 UTC m=+0.024785276 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:09:00.598 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:00 smithi033 podman[74143]: 2024-09-17 01:09:00.492072515 +0000 UTC m=+0.205930468 container create 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-17T01:09:00.851 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:00 smithi033 podman[74143]: 2024-09-17 01:09:00.83724525 +0000 UTC m=+0.551103207 container init 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:09:00.851 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:00 smithi033 podman[74143]: 2024-09-17 01:09:00.845424402 +0000 UTC m=+0.559282350 container start 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:09:00.996 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:00 smithi033 ceph-osd[74161]: -- 172.21.15.33:0/3714898468 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564b3f6321e0 con 0x564b3e827c00 2024-09-17T01:09:01.102 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:01 smithi033 bash[74143]: 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd 2024-09-17T01:09:01.102 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:01 smithi033 systemd[1]: Started Ceph osd.1 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:09:01.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 147310 8.69 MB/sec execute 300 sec latency 61.848 ms 2024-09-17T01:09:02.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:02 smithi033 ceph-mon[65430]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 441 op/s; 187/1665 objects degraded (11.231%) 2024-09-17T01:09:02.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:02 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:02.106 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:02 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:02.107 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:02 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:09:02.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 148088 8.71 MB/sec execute 301 sec latency 21.047 ms 2024-09-17T01:09:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:02 smithi100 ceph-mon[51701]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.7 MiB/s wr, 441 op/s; 187/1665 objects degraded (11.231%) 2024-09-17T01:09:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:02 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:02 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:02 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:09:02.863 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:02 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:09:02.691+0000 7f6ff6826640 -1 Falling back to public interface 2024-09-17T01:09:03.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 148593 8.73 MB/sec execute 302 sec latency 32.347 ms 2024-09-17T01:09:03.365 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:03 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 195/1671 objects degraded (11.670%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:03.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:03 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 195/1671 objects degraded (11.670%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:04.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 149124 8.77 MB/sec execute 303 sec latency 76.074 ms 2024-09-17T01:09:04.373 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:04 smithi033 ceph-mon[65430]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.5 MiB/s wr, 283 op/s; 195/1671 objects degraded (11.670%) 2024-09-17T01:09:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:04 smithi100 ceph-mon[51701]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 2.5 MiB/s wr, 283 op/s; 195/1671 objects degraded (11.670%) 2024-09-17T01:09:05.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 149461 8.78 MB/sec execute 304 sec latency 62.191 ms 2024-09-17T01:09:06.157 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:06 smithi033 ceph-mon[65430]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 411 op/s; 206/1737 objects degraded (11.860%) 2024-09-17T01:09:06.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 149770 8.78 MB/sec execute 305 sec latency 46.836 ms 2024-09-17T01:09:06.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:06 smithi100 ceph-mon[51701]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 411 op/s; 206/1737 objects degraded (11.860%) 2024-09-17T01:09:07.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 149975 8.76 MB/sec execute 306 sec latency 41.901 ms 2024-09-17T01:09:08.177 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:08 smithi033 ceph-mon[65430]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 390 op/s; 206/1734 objects degraded (11.880%) 2024-09-17T01:09:08.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 150195 8.76 MB/sec execute 307 sec latency 92.636 ms 2024-09-17T01:09:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:08 smithi100 ceph-mon[51701]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.0 MiB/s wr, 390 op/s; 206/1734 objects degraded (11.880%) 2024-09-17T01:09:09.213 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:09 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 206/1734 objects degraded (11.880%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:09.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 150430 8.74 MB/sec execute 308 sec latency 71.551 ms 2024-09-17T01:09:09.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:09 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 206/1734 objects degraded (11.880%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:09.985 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:09 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:09:09.768+0000 7f6ff6826640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T01:09:10.236 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:10 smithi033 ceph-mon[65430]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 583 op/s; 187/1644 objects degraded (11.375%) 2024-09-17T01:09:10.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 150668 8.73 MB/sec execute 309 sec latency 73.657 ms 2024-09-17T01:09:10.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:10 smithi100 ceph-mon[51701]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.1 KiB/s rd, 5.9 MiB/s wr, 583 op/s; 187/1644 objects degraded (11.375%) 2024-09-17T01:09:10.767 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:10 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:09:10.542+0000 7f6ff6826640 -1 osd.1 85 log_to_monitors true 2024-09-17T01:09:11.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 150832 8.71 MB/sec execute 310 sec latency 107.737 ms 2024-09-17T01:09:11.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:11 smithi033 ceph-mon[65430]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:09:11.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:11.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:11 smithi100 ceph-mon[51701]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T01:09:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:11.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:11.901 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:09:11 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:09:11.735+0000 7f6fea5c9640 -1 osd.1 85 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:09:12.322 INFO:teuthology.orchestra.run.smithi033.stdout: 5 150966 8.69 MB/sec execute 311 sec latency 198.940 ms 2024-09-17T01:09:12.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 478 op/s; 187/1641 objects degraded (11.395%) 2024-09-17T01:09:12.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:09:12.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: osdmap e88: 8 total, 7 up, 8 in 2024-09-17T01:09:12.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:09:12.423 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' 2024-09-17T01:09:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:12.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.0 MiB/s wr, 478 op/s; 187/1641 objects degraded (11.395%) 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: osdmap e88: 8 total, 7 up, 8 in 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546]' entity='osd.1' 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:12.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:13.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 151068 8.68 MB/sec execute 312 sec latency 639.467 ms 2024-09-17T01:09:13.346 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:13 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:09:13.347 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:13 smithi033 ceph-mon[65430]: osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546] boot 2024-09-17T01:09:13.347 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:13 smithi033 ceph-mon[65430]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T01:09:13.347 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:13 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:09:13.434 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:09:13.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:13 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:09:13.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:13 smithi100 ceph-mon[51701]: osd.1 [v2:172.21.15.33:6810/3223148546,v1:172.21.15.33:6811/3223148546] boot 2024-09-17T01:09:13.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:13 smithi100 ceph-mon[51701]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T01:09:13.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:13 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T01:09:14.200 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (4m) 3s ago 13m 25.9M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (98s) 3s ago 13m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (91s) 87s ago 13m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (4m) 3s ago 13m 53.3M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (7m) 3s ago 7m 36.3M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (7m) 87s ago 7m 14.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (4m) 3s ago 14m 502M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:09:14.201 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (3m) 87s ago 12m 440M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (2m) 3s ago 14m 48.2M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (2m) 87s ago 12m 36.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (7m) 3s ago 7m 273M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (7m) 87s ago 7m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (4m) 3s ago 13m 8242k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (4m) 87s ago 12m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (60s) 3s ago 12m 212M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:09:14.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (13s) 3s ago 11m 11.8M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (11m) 3s ago 11m 337M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (10m) 3s ago 10m 491M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (10m) 87s ago 10m 449M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (10m) 87s ago 10m 491M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (9m) 87s ago 9m 409M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (9m) 87s ago 9m 572M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:09:14.203 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (3m) 3s ago 12m 56.8M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:09:14.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 151068 8.66 MB/sec execute 313 sec latency 1639.510 ms 2024-09-17T01:09:14.675 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:09:14.675 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:14.676 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:09:14.677 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T01:09:14.677 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T01:09:14.677 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:09:14.677 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:09:14.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:14 smithi100 ceph-mon[51701]: pgmap v103: 97 pgs: 4 peering, 14 active+undersized, 21 active+undersized+degraded, 58 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 340 op/s; 170/1629 objects degraded (10.436%) 2024-09-17T01:09:14.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:14 smithi100 ceph-mon[51701]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:14.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:14 smithi100 ceph-mon[51701]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T01:09:14.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:14 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 170/1629 objects degraded (10.436%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:14.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:14 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1844153147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:15.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:14 smithi033 ceph-mon[65430]: pgmap v103: 97 pgs: 4 peering, 14 active+undersized, 21 active+undersized+degraded, 58 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.1 MiB/s wr, 340 op/s; 170/1629 objects degraded (10.436%) 2024-09-17T01:09:15.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:14 smithi033 ceph-mon[65430]: from='client.34256 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:15.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:14 smithi033 ceph-mon[65430]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T01:09:15.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:14 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 170/1629 objects degraded (10.436%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:15.049 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:14 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1844153147' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:15.059 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:09:15.060 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:09:15.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 151107 8.63 MB/sec execute 314 sec latency 2367.889 ms 2024-09-17T01:09:15.682 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN Degraded data redundancy: 170/1629 objects degraded (10.436%), 21 pgs degraded 2024-09-17T01:09:15.682 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 170/1629 objects degraded (10.436%), 21 pgs degraded 2024-09-17T01:09:15.682 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-17T01:09:15.682 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-17T01:09:15.682 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-17T01:09:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-17T01:09:15.684 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-17T01:09:15.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:15 smithi033 ceph-mon[65430]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:15.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:15 smithi033 ceph-mon[65430]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:15.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:15 smithi033 ceph-mon[65430]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T01:09:15.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:15 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1889156587' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:09:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:15 smithi100 ceph-mon[51701]: from='client.34260 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:15 smithi100 ceph-mon[51701]: from='client.34264 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:15 smithi100 ceph-mon[51701]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T01:09:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:15 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1889156587' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:09:16.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 151236 8.61 MB/sec execute 315 sec latency 615.917 ms 2024-09-17T01:09:16.834 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: pgmap v106: 97 pgs: 25 peering, 4 active+undersized, 10 active+undersized+degraded, 58 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.6 MiB/s wr, 340 op/s; 85/1530 objects degraded (5.556%) 2024-09-17T01:09:16.834 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.835 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: pgmap v106: 97 pgs: 25 peering, 4 active+undersized, 10 active+undersized+degraded, 58 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.6 MiB/s wr, 340 op/s; 85/1530 objects degraded (5.556%) 2024-09-17T01:09:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='client.34272 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T01:09:16.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:09:16.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:09:16.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:16.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:17.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 151585 8.61 MB/sec execute 316 sec latency 1031.572 ms 2024-09-17T01:09:17.838 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T01:09:17.838 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:09:17.838 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:17.838 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:17.839 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:17.839 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:17.839 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:17.839 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T01:09:17.839 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:17 smithi033 ceph-mon[65430]: pgmap v109: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 25 peering, 2 active+undersized+degraded, 69 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.9 MiB/s wr, 356 op/s; 56/1509 objects degraded (3.711%); 22 B/s, 62 keys/s, 5 objects/s recovering 2024-09-17T01:09:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T01:09:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:09:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:17.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:17.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:17.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:17.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T01:09:17.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:17 smithi100 ceph-mon[51701]: pgmap v109: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 25 peering, 2 active+undersized+degraded, 69 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 2.9 MiB/s wr, 356 op/s; 56/1509 objects degraded (3.711%); 22 B/s, 62 keys/s, 5 objects/s recovering 2024-09-17T01:09:18.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 152019 8.60 MB/sec execute 317 sec latency 415.918 ms 2024-09-17T01:09:18.856 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:18 smithi033 ceph-mon[65430]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T01:09:19.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:18 smithi100 ceph-mon[51701]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T01:09:19.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 152232 8.60 MB/sec execute 318 sec latency 850.922 ms 2024-09-17T01:09:19.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:19 smithi033 ceph-mon[65430]: osdmap e95: 8 total, 8 up, 8 in 2024-09-17T01:09:19.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:19 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 56/1509 objects degraded (3.711%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:19.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:19 smithi033 ceph-mon[65430]: pgmap v112: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 91 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.4 MiB/s wr, 276 op/s; 88/1572 objects degraded (5.598%); 2.4 MiB/s, 62 keys/s, 36 objects/s recovering 2024-09-17T01:09:19.907 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:19 smithi033 ceph-mon[65430]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T01:09:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:19 smithi100 ceph-mon[51701]: osdmap e95: 8 total, 8 up, 8 in 2024-09-17T01:09:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:19 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 56/1509 objects degraded (3.711%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:19 smithi100 ceph-mon[51701]: pgmap v112: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 91 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.4 MiB/s wr, 276 op/s; 88/1572 objects degraded (5.598%); 2.4 MiB/s, 62 keys/s, 36 objects/s recovering 2024-09-17T01:09:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:19 smithi100 ceph-mon[51701]: osdmap e96: 8 total, 8 up, 8 in 2024-09-17T01:09:20.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 152527 8.62 MB/sec execute 319 sec latency 726.518 ms 2024-09-17T01:09:21.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 152916 8.62 MB/sec execute 320 sec latency 54.175 ms 2024-09-17T01:09:21.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:21 smithi033 ceph-mon[65430]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T01:09:21.961 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:21 smithi033 ceph-mon[65430]: pgmap v115: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 92 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 285 op/s; 66/1605 objects degraded (4.112%); 2.5 MiB/s, 36 objects/s recovering 2024-09-17T01:09:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:21 smithi100 ceph-mon[51701]: osdmap e97: 8 total, 8 up, 8 in 2024-09-17T01:09:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:21 smithi100 ceph-mon[51701]: pgmap v115: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+undersized+degraded, 92 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 285 op/s; 66/1605 objects degraded (4.112%); 2.5 MiB/s, 36 objects/s recovering 2024-09-17T01:09:22.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 153361 8.63 MB/sec execute 321 sec latency 63.619 ms 2024-09-17T01:09:23.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 153891 8.65 MB/sec execute 322 sec latency 39.811 ms 2024-09-17T01:09:24.278 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:24 smithi033 ceph-mon[65430]: pgmap v116: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 95 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.2 MiB/s wr, 272 op/s; 4/1629 objects degraded (0.246%); 1.9 MiB/s, 41 objects/s recovering 2024-09-17T01:09:24.278 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:24 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 4/1629 objects degraded (0.246%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:09:24.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 154220 8.66 MB/sec execute 323 sec latency 69.777 ms 2024-09-17T01:09:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:24 smithi100 ceph-mon[51701]: pgmap v116: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 95 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.2 MiB/s wr, 272 op/s; 4/1629 objects degraded (0.246%); 1.9 MiB/s, 41 objects/s recovering 2024-09-17T01:09:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:24 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 4/1629 objects degraded (0.246%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:09:25.290 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:25 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/1629 objects degraded (0.246%), 1 pg degraded) 2024-09-17T01:09:25.290 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:25 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:09:25.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 154664 8.65 MB/sec execute 324 sec latency 53.706 ms 2024-09-17T01:09:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:25 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/1629 objects degraded (0.246%), 1 pg degraded) 2024-09-17T01:09:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:25 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:09:26.305 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:26 smithi033 ceph-mon[65430]: pgmap v117: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 382 op/s; 51 KiB/s, 16 objects/s recovering 2024-09-17T01:09:26.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 154908 8.64 MB/sec execute 325 sec latency 88.954 ms 2024-09-17T01:09:26.378 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:26 smithi100 ceph-mon[51701]: pgmap v117: 97 pgs: 97 active+clean; 341 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 382 op/s; 51 KiB/s, 16 objects/s recovering 2024-09-17T01:09:27.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 155307 8.65 MB/sec execute 326 sec latency 38.507 ms 2024-09-17T01:09:28.125 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:28 smithi033 ceph-mon[65430]: pgmap v118: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.9 MiB/s wr, 453 op/s; 40 KiB/s, 12 objects/s recovering 2024-09-17T01:09:28.125 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:28.125 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:28.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 155644 8.64 MB/sec execute 327 sec latency 43.962 ms 2024-09-17T01:09:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:28 smithi100 ceph-mon[51701]: pgmap v118: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.9 MiB/s wr, 453 op/s; 40 KiB/s, 12 objects/s recovering 2024-09-17T01:09:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:29.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 155998 8.66 MB/sec execute 328 sec latency 52.625 ms 2024-09-17T01:09:30.069 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:30 smithi033 ceph-mon[65430]: pgmap v119: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 34 KiB/s, 11 objects/s recovering 2024-09-17T01:09:30.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 156304 8.66 MB/sec execute 329 sec latency 42.148 ms 2024-09-17T01:09:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:30 smithi100 ceph-mon[51701]: pgmap v119: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.7 MiB/s wr, 571 op/s; 34 KiB/s, 11 objects/s recovering 2024-09-17T01:09:31.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 156585 8.66 MB/sec execute 330 sec latency 61.915 ms 2024-09-17T01:09:32.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:32 smithi033 ceph-mon[65430]: pgmap v120: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 516 op/s; 0 B/s, 7 objects/s recovering 2024-09-17T01:09:32.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:32.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 156866 8.65 MB/sec execute 331 sec latency 44.506 ms 2024-09-17T01:09:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:32 smithi100 ceph-mon[51701]: pgmap v120: 97 pgs: 97 active+clean; 337 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.2 MiB/s wr, 516 op/s; 0 B/s, 7 objects/s recovering 2024-09-17T01:09:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:33.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:33 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:33.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:33 smithi033 ceph-mon[65430]: Upgrade: osd.2 is safe to restart 2024-09-17T01:09:33.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:33 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:33.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 157123 8.66 MB/sec execute 332 sec latency 37.867 ms 2024-09-17T01:09:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:33 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T01:09:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:33 smithi100 ceph-mon[51701]: Upgrade: osd.2 is safe to restart 2024-09-17T01:09:33.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:33 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:34.138 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:34 smithi033 ceph-mon[65430]: pgmap v121: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 567 op/s; 0 B/s, 6 objects/s recovering 2024-09-17T01:09:34.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 157346 8.65 MB/sec execute 333 sec latency 82.184 ms 2024-09-17T01:09:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:34 smithi100 ceph-mon[51701]: pgmap v121: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.5 MiB/s wr, 567 op/s; 0 B/s, 6 objects/s recovering 2024-09-17T01:09:35.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 157641 8.65 MB/sec execute 334 sec latency 67.308 ms 2024-09-17T01:09:35.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:35 smithi033 ceph-mon[65430]: Upgrade: Updating osd.2 2024-09-17T01:09:35.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:35 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:35.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:35 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:09:35.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:35 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:09:35.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:35 smithi033 ceph-mon[65430]: Deploying daemon osd.2 on smithi033 2024-09-17T01:09:35.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:35 smithi100 ceph-mon[51701]: Upgrade: Updating osd.2 2024-09-17T01:09:35.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:35 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:35.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:35 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T01:09:35.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:35 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:09:35.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:35 smithi100 ceph-mon[51701]: Deploying daemon osd.2 on smithi033 2024-09-17T01:09:36.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 157876 8.63 MB/sec execute 335 sec latency 98.453 ms 2024-09-17T01:09:36.702 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:36 smithi033 ceph-mon[65430]: pgmap v122: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 591 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T01:09:36.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:36 smithi100 ceph-mon[51701]: pgmap v122: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 591 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T01:09:37.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 158263 8.62 MB/sec execute 336 sec latency 48.777 ms 2024-09-17T01:09:38.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 158602 8.63 MB/sec execute 337 sec latency 53.950 ms 2024-09-17T01:09:38.501 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:38 smithi033 ceph-mon[65430]: pgmap v123: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-09-17T01:09:38.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:38 smithi100 ceph-mon[51701]: pgmap v123: 97 pgs: 97 active+clean; 339 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.4 MiB/s wr, 442 op/s 2024-09-17T01:09:39.265 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:39 smithi033 systemd[1]: Stopping Ceph osd.2 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:09:39.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 159056 8.64 MB/sec execute 338 sec latency 49.372 ms 2024-09-17T01:09:39.769 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:39 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[41913]: 2024-09-17T01:09:39.551+0000 7f36d7d69700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:09:39.770 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:39 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[41913]: 2024-09-17T01:09:39.551+0000 7f36d7d69700 -1 osd.2 97 *** Got signal Terminated *** 2024-09-17T01:09:39.770 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:39 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[41913]: 2024-09-17T01:09:39.551+0000 7f36d7d69700 -1 osd.2 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:09:40.323 INFO:teuthology.orchestra.run.smithi033.stdout: 5 159443 8.64 MB/sec execute 339 sec latency 61.429 ms 2024-09-17T01:09:40.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:40 smithi033 ceph-mon[65430]: pgmap v124: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 505 op/s 2024-09-17T01:09:40.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:40 smithi033 ceph-mon[65430]: osd.2 marked itself down and dead 2024-09-17T01:09:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:40 smithi100 ceph-mon[51701]: pgmap v124: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 4.9 MiB/s wr, 505 op/s 2024-09-17T01:09:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:40 smithi100 ceph-mon[51701]: osd.2 marked itself down and dead 2024-09-17T01:09:41.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 159514 8.64 MB/sec execute 340 sec latency 864.136 ms 2024-09-17T01:09:41.405 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:41 smithi033 podman[76267]: 2024-09-17 01:09:41.125926528 +0000 UTC m=+1.711335854 container died 0f139bf6a3fe04ebbe177aadfedbf00e8a7100e3a66a4aae7b825358a188cd8e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, ceph=True, RELEASE=HEAD, vcs-type=git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, distribution-scope=public, io.openshift.expose-services=, name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T01:09:41.657 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:41 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:09:41.657 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:41 smithi033 ceph-mon[65430]: osdmap e98: 8 total, 7 up, 8 in 2024-09-17T01:09:41.657 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:41 smithi033 podman[76267]: 2024-09-17 01:09:41.514560793 +0000 UTC m=+2.099970122 container cleanup 0f139bf6a3fe04ebbe177aadfedbf00e8a7100e3a66a4aae7b825358a188cd8e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, release=754, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, version=8, distribution-scope=public, ceph=True, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T01:09:41.657 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:41 smithi033 bash[76267]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2 2024-09-17T01:09:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:41 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:09:41.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:41 smithi100 ceph-mon[51701]: osdmap e98: 8 total, 7 up, 8 in 2024-09-17T01:09:41.908 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:41 smithi033 podman[76314]: 2024-09-17 01:09:41.794763938 +0000 UTC m=+0.670432367 container remove 0f139bf6a3fe04ebbe177aadfedbf00e8a7100e3a66a4aae7b825358a188cd8e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, version=8, architecture=x86_64, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux ) 2024-09-17T01:09:42.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 160076 8.66 MB/sec execute 341 sec latency 1041.054 ms 2024-09-17T01:09:42.410 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:42 smithi033 podman[76399]: 2024-09-17 01:09:42.282412631 +0000 UTC m=+0.027992938 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:09:42.410 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:42 smithi033 podman[76399]: 2024-09-17 01:09:42.401748523 +0000 UTC m=+0.147328828 container create e721ef72fae2e492d362281ce9da55778c8563d335f7d5534c9aa654c8015282 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97) 2024-09-17T01:09:42.665 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:42 smithi033 ceph-mon[65430]: pgmap v126: 97 pgs: 6 stale+active+clean, 91 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.6 MiB/s wr, 444 op/s 2024-09-17T01:09:42.665 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:42 smithi033 ceph-mon[65430]: osdmap e99: 8 total, 7 up, 8 in 2024-09-17T01:09:42.665 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:42.665 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:42 smithi100 ceph-mon[51701]: pgmap v126: 97 pgs: 6 stale+active+clean, 91 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.6 MiB/s wr, 444 op/s 2024-09-17T01:09:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:42 smithi100 ceph-mon[51701]: osdmap e99: 8 total, 7 up, 8 in 2024-09-17T01:09:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:43.169 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:42 smithi033 podman[76399]: 2024-09-17 01:09:42.920265851 +0000 UTC m=+0.665846169 container init e721ef72fae2e492d362281ce9da55778c8563d335f7d5534c9aa654c8015282 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T01:09:43.169 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:42 smithi033 podman[76399]: 2024-09-17 01:09:42.92949032 +0000 UTC m=+0.675070633 container start e721ef72fae2e492d362281ce9da55778c8563d335f7d5534c9aa654c8015282 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:09:43.169 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:43 smithi033 podman[76399]: 2024-09-17 01:09:43.097554782 +0000 UTC m=+0.843135089 container attach e721ef72fae2e492d362281ce9da55778c8563d335f7d5534c9aa654c8015282 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T01:09:43.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 160616 8.68 MB/sec execute 342 sec latency 32.911 ms 2024-09-17T01:09:43.419 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:43 smithi033 podman[76399]: 2024-09-17 01:09:43.283134572 +0000 UTC m=+1.028714893 container died e721ef72fae2e492d362281ce9da55778c8563d335f7d5534c9aa654c8015282 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T01:09:43.670 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:43 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 68/1641 objects degraded (4.144%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:43 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 68/1641 objects degraded (4.144%), 9 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:44.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 161022 8.68 MB/sec execute 343 sec latency 30.925 ms 2024-09-17T01:09:44.430 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 podman[76421]: 2024-09-17 01:09:44.21477113 +0000 UTC m=+0.924005124 container remove e721ef72fae2e492d362281ce9da55778c8563d335f7d5534c9aa654c8015282 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T01:09:44.430 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T01:09:44.430 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 systemd[1]: Stopped Ceph osd.2 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:09:44.431 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.2.service: Consumed 1min 43.072s CPU time. 2024-09-17T01:09:44.684 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:44 smithi033 ceph-mon[65430]: pgmap v128: 97 pgs: 2 active+undersized, 5 peering, 3 stale+active+clean, 9 active+undersized+degraded, 78 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.3 MiB/s wr, 424 op/s; 68/1641 objects degraded (4.144%) 2024-09-17T01:09:44.684 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 systemd[1]: Starting Ceph osd.2 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:09:44.936 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 podman[76518]: 2024-09-17 01:09:44.743546428 +0000 UTC m=+0.024399217 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:09:44.936 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:44 smithi033 podman[76518]: 2024-09-17 01:09:44.882632195 +0000 UTC m=+0.163484966 container create 2b41a34cfd5b56f55f9ca8671b4c2f44f6d0fa48f0b86aba98a6b6645be78d3f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD) 2024-09-17T01:09:44.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:44 smithi100 ceph-mon[51701]: pgmap v128: 97 pgs: 2 active+undersized, 5 peering, 3 stale+active+clean, 9 active+undersized+degraded, 78 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.3 MiB/s wr, 424 op/s; 68/1641 objects degraded (4.144%) 2024-09-17T01:09:45.189 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 podman[76518]: 2024-09-17 01:09:45.186089121 +0000 UTC m=+0.466941898 container init 2b41a34cfd5b56f55f9ca8671b4c2f44f6d0fa48f0b86aba98a6b6645be78d3f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-17T01:09:45.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 161394 8.69 MB/sec execute 344 sec latency 62.544 ms 2024-09-17T01:09:45.442 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 podman[76518]: 2024-09-17 01:09:45.192176779 +0000 UTC m=+0.473029558 container start 2b41a34cfd5b56f55f9ca8671b4c2f44f6d0fa48f0b86aba98a6b6645be78d3f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T01:09:45.444 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 podman[76518]: 2024-09-17 01:09:45.350342028 +0000 UTC m=+0.631194798 container attach 2b41a34cfd5b56f55f9ca8671b4c2f44f6d0fa48f0b86aba98a6b6645be78d3f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-17T01:09:45.444 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:09:45.444 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 bash[76518]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:09:45.444 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:09:45.444 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:45 smithi033 bash[76518]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:09:46.230 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:09:46.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 161743 8.67 MB/sec execute 345 sec latency 56.927 ms 2024-09-17T01:09:46.479 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:09:46.479 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:09:46.479 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:09:46.479 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:09:46.730 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-mon[65430]: pgmap v129: 97 pgs: 3 active+undersized, 5 peering, 15 active+undersized+degraded, 74 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 581 op/s; 106/1707 objects degraded (6.210%) 2024-09-17T01:09:46.730 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:09:46.730 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:09:46.730 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate[76533]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T01:09:46.731 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 bash[76518]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T01:09:46.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:46 smithi100 ceph-mon[51701]: pgmap v129: 97 pgs: 3 active+undersized, 5 peering, 15 active+undersized+degraded, 74 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.5 MiB/s wr, 581 op/s; 106/1707 objects degraded (6.210%) 2024-09-17T01:09:46.982 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:46 smithi033 podman[76518]: 2024-09-17 01:09:46.738006747 +0000 UTC m=+2.018859522 container died 2b41a34cfd5b56f55f9ca8671b4c2f44f6d0fa48f0b86aba98a6b6645be78d3f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (5m) 36s ago 14m 25.9M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (2m) 36s ago 14m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (2m) 2m ago 13m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (5m) 36s ago 13m 53.3M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (8m) 36s ago 8m 36.3M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:09:47.181 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (8m) 2m ago 8m 14.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (4m) 36s ago 15m 502M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (3m) 2m ago 13m 440M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (3m) 36s ago 15m 48.2M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (3m) 2m ago 13m 36.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (7m) 36s ago 7m 273M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (7m) 2m ago 7m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (5m) 36s ago 13m 8242k - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (4m) 2m ago 13m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (93s) 36s ago 12m 212M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (46s) 36s ago 12m 11.8M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (11m) 36s ago 11m 337M 1937M 17.2.0 e1d6a67b021e 0f139bf6a3fe 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (11m) 36s ago 11m 491M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (10m) 2m ago 10m 449M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (10m) 2m ago 10m 491M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:09:47.182 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (10m) 2m ago 10m 409M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:09:47.183 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (9m) 2m ago 9m 572M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:09:47.183 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (3m) 36s ago 13m 56.8M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:09:47.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 162023 8.67 MB/sec execute 346 sec latency 60.675 ms 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:09:47.723 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:09:47.724 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:09:47.743 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:47 smithi033 ceph-mon[65430]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:47 smithi100 ceph-mon[51701]: from='client.44187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:47.994 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:47 smithi033 podman[76727]: 2024-09-17 01:09:47.921221874 +0000 UTC m=+1.174764467 container remove 2b41a34cfd5b56f55f9ca8671b4c2f44f6d0fa48f0b86aba98a6b6645be78d3f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T01:09:48.182 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:09:48.182 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:09:48.182 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:09:48.183 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:09:48.247 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:48 smithi033 podman[76847]: 2024-09-17 01:09:48.081818275 +0000 UTC m=+0.022564136 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:09:48.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 162255 8.66 MB/sec execute 347 sec latency 78.783 ms 2024-09-17T01:09:48.498 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:48 smithi033 podman[76847]: 2024-09-17 01:09:48.486879622 +0000 UTC m=+0.427625471 container create e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T01:09:48.751 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:48 smithi033 ceph-mon[65430]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:48.751 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:48 smithi033 ceph-mon[65430]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.2 MiB/s wr, 361 op/s; 156/1728 objects degraded (9.028%) 2024-09-17T01:09:48.751 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:48 smithi033 ceph-mon[65430]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:48.751 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:48 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/4231521578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:48.839 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 156/1728 objects degraded (9.028%), 19 pgs degraded 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: osd.2 (root=default,host=smithi033) is down 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 156/1728 objects degraded (9.028%), 19 pgs degraded 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-17T01:09:48.840 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-17T01:09:48.841 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-17T01:09:48.842 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-17T01:09:48.842 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-17T01:09:48.842 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-17T01:09:48.842 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-17T01:09:48.842 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-17T01:09:48.842 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-17T01:09:48.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:48 smithi100 ceph-mon[51701]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:48.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:48 smithi100 ceph-mon[51701]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 4.2 MiB/s wr, 361 op/s; 156/1728 objects degraded (9.028%) 2024-09-17T01:09:48.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:48 smithi100 ceph-mon[51701]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:48.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:48 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/4231521578' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:09:49.014 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:48 smithi033 podman[76847]: 2024-09-17 01:09:48.882935674 +0000 UTC m=+0.823681535 container init e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T01:09:49.014 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:48 smithi033 podman[76847]: 2024-09-17 01:09:48.89476127 +0000 UTC m=+0.835507122 container start e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T01:09:49.014 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:48 smithi033 ceph-osd[76890]: -- 172.21.15.33:0/1351787462 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5590453301e0 con 0x55904451dc00 2024-09-17T01:09:49.264 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:49 smithi033 bash[76847]: e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 2024-09-17T01:09:49.265 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:49 smithi033 systemd[1]: Started Ceph osd.2 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:09:49.324 INFO:teuthology.orchestra.run.smithi033.stdout: 5 162676 8.67 MB/sec execute 348 sec latency 48.508 ms 2024-09-17T01:09:49.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:49 smithi033 ceph-mon[65430]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:49.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:49 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2137909606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:09:49.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:49 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 156/1728 objects degraded (9.028%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:49.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:49 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:49.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:49 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:49.770 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:49 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:09:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:49 smithi100 ceph-mon[51701]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:09:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:49 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2137909606' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:09:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:49 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 156/1728 objects degraded (9.028%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:49 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:49 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:49 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:09:50.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 162992 8.67 MB/sec execute 349 sec latency 42.982 ms 2024-09-17T01:09:50.793 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:50 smithi033 ceph-mon[65430]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.2 MiB/s wr, 619 op/s; 137/1632 objects degraded (8.395%) 2024-09-17T01:09:50.794 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:50 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:09:50.768+0000 7f4c2f858640 -1 Falling back to public interface 2024-09-17T01:09:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:50 smithi100 ceph-mon[51701]: pgmap v131: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 6.2 MiB/s wr, 619 op/s; 137/1632 objects degraded (8.395%) 2024-09-17T01:09:51.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 163382 8.69 MB/sec execute 350 sec latency 59.851 ms 2024-09-17T01:09:52.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 163685 8.69 MB/sec execute 351 sec latency 53.424 ms 2024-09-17T01:09:52.571 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:52 smithi033 ceph-mon[65430]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.6 MiB/s wr, 544 op/s; 138/1608 objects degraded (8.582%) 2024-09-17T01:09:52.571 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:52.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:52 smithi100 ceph-mon[51701]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.6 MiB/s wr, 544 op/s; 138/1608 objects degraded (8.582%) 2024-09-17T01:09:52.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:53.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 163945 8.69 MB/sec execute 352 sec latency 74.249 ms 2024-09-17T01:09:54.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 164199 8.68 MB/sec execute 353 sec latency 58.399 ms 2024-09-17T01:09:54.602 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:54 smithi033 ceph-mon[65430]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.6 MiB/s wr, 596 op/s; 145/1623 objects degraded (8.934%) 2024-09-17T01:09:54.602 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:54 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 145/1623 objects degraded (8.934%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:54.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:54 smithi100 ceph-mon[51701]: pgmap v133: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 5.6 MiB/s wr, 596 op/s; 145/1623 objects degraded (8.934%) 2024-09-17T01:09:54.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:54 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 145/1623 objects degraded (8.934%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:55.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 164397 8.67 MB/sec execute 354 sec latency 57.098 ms 2024-09-17T01:09:56.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 164616 8.65 MB/sec execute 355 sec latency 61.766 ms 2024-09-17T01:09:56.727 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:56 smithi033 ceph-mon[65430]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 525 op/s; 152/1671 objects degraded (9.096%) 2024-09-17T01:09:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:56 smithi100 ceph-mon[51701]: pgmap v134: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 342 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 525 op/s; 152/1671 objects degraded (9.096%) 2024-09-17T01:09:57.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 164847 8.66 MB/sec execute 356 sec latency 82.500 ms 2024-09-17T01:09:57.498 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:57 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:09:57.438+0000 7f4c2f858640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T01:09:58.016 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:58 smithi033 ceph-mon[65430]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 437 op/s; 151/1668 objects degraded (9.053%) 2024-09-17T01:09:58.016 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:58.017 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:58.268 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:58.268 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:58.268 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:58 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:09:58.159+0000 7f4c2f858640 -1 osd.2 97 log_to_monitors true 2024-09-17T01:09:58.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 165037 8.64 MB/sec execute 357 sec latency 82.804 ms 2024-09-17T01:09:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:58 smithi100 ceph-mon[51701]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.6 MiB/s wr, 437 op/s; 151/1668 objects degraded (9.053%) 2024-09-17T01:09:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:09:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:59.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:59 smithi033 ceph-mon[65430]: from='osd.2 [v2:172.21.15.33:6818/2545287228,v1:172.21.15.33:6819/2545287228]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:09:59.033 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:59 smithi033 ceph-mon[65430]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:09:59.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:59 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:59.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:59 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:59.034 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:09:59 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 151/1668 objects degraded (9.053%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:09:59.291 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:09:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:09:59.100+0000 7f4c27603640 -1 osd.2 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:09:59.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 165328 8.63 MB/sec execute 358 sec latency 47.240 ms 2024-09-17T01:09:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:59 smithi100 ceph-mon[51701]: from='osd.2 [v2:172.21.15.33:6818/2545287228,v1:172.21.15.33:6819/2545287228]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:09:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:59 smithi100 ceph-mon[51701]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T01:09:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:59 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:59 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:09:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:09:59 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 151/1668 objects degraded (9.053%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.3 MiB/s wr, 554 op/s; 141/1623 objects degraded (8.688%) 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: from='osd.2 [v2:172.21.15.33:6818/2545287228,v1:172.21.15.33:6819/2545287228]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: osdmap e100: 8 total, 7 up, 8 in 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 141/1623 objects degraded (8.688%), 19 pgs degraded 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: [WRN] OSD_DOWN: 1 osds down 2024-09-17T01:10:00.071 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: osd.2 (root=default,host=smithi033) is down 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: [WRN] PG_DEGRADED: Degraded data redundancy: 141/1623 objects degraded (8.688%), 19 pgs degraded 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-17T01:10:00.072 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-17T01:10:00.073 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:00 smithi033 ceph-mon[65430]: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-17T01:10:00.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 165521 8.63 MB/sec execute 359 sec latency 110.037 ms 2024-09-17T01:10:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pgmap v136: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.3 MiB/s wr, 554 op/s; 141/1623 objects degraded (8.688%) 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: from='osd.2 [v2:172.21.15.33:6818/2545287228,v1:172.21.15.33:6819/2545287228]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: osdmap e100: 8 total, 7 up, 8 in 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: Health detail: HEALTH_WARN 1 osds down; Degraded data redundancy: 141/1623 objects degraded (8.688%), 19 pgs degraded 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: [WRN] OSD_DOWN: 1 osds down 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: osd.2 (root=default,host=smithi033) is down 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: [WRN] PG_DEGRADED: Degraded data redundancy: 141/1623 objects degraded (8.688%), 19 pgs degraded 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-17T01:10:00.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.f is active+undersized+degraded, acting [4,0] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-17T01:10:00.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-17T01:10:00.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:00 smithi100 ceph-mon[51701]: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-17T01:10:01.097 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:01 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:10:01.097 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:01 smithi033 ceph-mon[65430]: osd.2 [v2:172.21.15.33:6818/2545287228,v1:172.21.15.33:6819/2545287228] boot 2024-09-17T01:10:01.097 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:01 smithi033 ceph-mon[65430]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T01:10:01.097 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:10:01.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 165700 8.61 MB/sec execute 360 sec latency 866.073 ms 2024-09-17T01:10:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:01 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:10:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:01 smithi100 ceph-mon[51701]: osd.2 [v2:172.21.15.33:6818/2545287228,v1:172.21.15.33:6819/2545287228] boot 2024-09-17T01:10:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:01 smithi100 ceph-mon[51701]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T01:10:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T01:10:02.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:02 smithi033 ceph-mon[65430]: pgmap v139: 97 pgs: 3 peering, 5 active+undersized, 18 active+undersized+degraded, 71 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.8 MiB/s wr, 309 op/s; 125/1569 objects degraded (7.967%) 2024-09-17T01:10:02.288 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:02 smithi033 ceph-mon[65430]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T01:10:02.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 165805 8.59 MB/sec execute 361 sec latency 1187.754 ms 2024-09-17T01:10:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:02 smithi100 ceph-mon[51701]: pgmap v139: 97 pgs: 3 peering, 5 active+undersized, 18 active+undersized+degraded, 71 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.8 MiB/s wr, 309 op/s; 125/1569 objects degraded (7.967%) 2024-09-17T01:10:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:02 smithi100 ceph-mon[51701]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T01:10:03.061 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:03 smithi033 ceph-mon[65430]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T01:10:03.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 166138 8.59 MB/sec execute 362 sec latency 291.417 ms 2024-09-17T01:10:03.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:03 smithi100 ceph-mon[51701]: osdmap e103: 8 total, 8 up, 8 in 2024-09-17T01:10:04.111 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:04 smithi033 ceph-mon[65430]: pgmap v142: 97 pgs: 4 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 2 active+undersized, 4 active+undersized+degraded, 82 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s wr, 400 op/s; 38/1485 objects degraded (2.559%); 1.6 MiB/s, 25 objects/s recovering 2024-09-17T01:10:04.111 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:04 smithi033 ceph-mon[65430]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T01:10:04.112 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:04 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 38/1485 objects degraded (2.559%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:04.325 INFO:teuthology.orchestra.run.smithi033.stdout: 5 166389 8.59 MB/sec execute 363 sec latency 1140.467 ms 2024-09-17T01:10:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:04 smithi100 ceph-mon[51701]: pgmap v142: 97 pgs: 4 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 2 active+undersized, 4 active+undersized+degraded, 82 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 MiB/s wr, 400 op/s; 38/1485 objects degraded (2.559%); 1.6 MiB/s, 25 objects/s recovering 2024-09-17T01:10:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:04 smithi100 ceph-mon[51701]: osdmap e104: 8 total, 8 up, 8 in 2024-09-17T01:10:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:04 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 38/1485 objects degraded (2.559%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:05.135 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T01:10:05.135 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.136 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.136 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:10:05.136 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:10:05.136 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.136 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.137 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:10:05.137 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:05.137 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:05.137 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:05.137 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:05 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:05.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 166717 8.59 MB/sec execute 364 sec latency 38.532 ms 2024-09-17T01:10:05.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: osdmap e105: 8 total, 8 up, 8 in 2024-09-17T01:10:05.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:05.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:05 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:06.176 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:06 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:06.176 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:06 smithi033 ceph-mon[65430]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T01:10:06.176 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:06 smithi033 ceph-mon[65430]: pgmap v145: 97 pgs: 4 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 88 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.0 MiB/s wr, 382 op/s; 21/1485 objects degraded (1.414%); 1.7 MiB/s, 25 objects/s recovering 2024-09-17T01:10:06.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 166879 8.58 MB/sec execute 365 sec latency 53.130 ms 2024-09-17T01:10:06.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:06 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:06.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:06 smithi100 ceph-mon[51701]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T01:10:06.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:06 smithi100 ceph-mon[51701]: pgmap v145: 97 pgs: 4 active+undersized+remapped, 1 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 88 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.0 MiB/s wr, 382 op/s; 21/1485 objects degraded (1.414%); 1.7 MiB/s, 25 objects/s recovering 2024-09-17T01:10:07.220 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:07 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1485 objects degraded (1.414%), 1 pg degraded) 2024-09-17T01:10:07.220 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:07 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:10:07.220 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:07.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 167220 8.59 MB/sec execute 366 sec latency 78.784 ms 2024-09-17T01:10:07.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:07 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 21/1485 objects degraded (1.414%), 1 pg degraded) 2024-09-17T01:10:07.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:07 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:10:07.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:08.231 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:08 smithi033 ceph-mon[65430]: pgmap v146: 97 pgs: 4 active+undersized+remapped, 93 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.5 MiB/s wr, 284 op/s; 1.2 MiB/s, 27 objects/s recovering 2024-09-17T01:10:08.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 167641 8.60 MB/sec execute 367 sec latency 45.911 ms 2024-09-17T01:10:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:08 smithi100 ceph-mon[51701]: pgmap v146: 97 pgs: 4 active+undersized+remapped, 93 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.5 MiB/s wr, 284 op/s; 1.2 MiB/s, 27 objects/s recovering 2024-09-17T01:10:09.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 167959 8.61 MB/sec execute 368 sec latency 98.715 ms 2024-09-17T01:10:10.273 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:10 smithi033 ceph-mon[65430]: pgmap v147: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.6 MiB/s wr, 408 op/s; 1.0 MiB/s, 23 objects/s recovering 2024-09-17T01:10:10.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 168257 8.61 MB/sec execute 369 sec latency 43.976 ms 2024-09-17T01:10:10.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:10 smithi100 ceph-mon[51701]: pgmap v147: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.6 MiB/s wr, 408 op/s; 1.0 MiB/s, 23 objects/s recovering 2024-09-17T01:10:11.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 168563 8.59 MB/sec execute 370 sec latency 44.550 ms 2024-09-17T01:10:12.320 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:12 smithi033 ceph-mon[65430]: pgmap v148: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.6 MiB/s wr, 317 op/s; 79 KiB/s, 7 objects/s recovering 2024-09-17T01:10:12.320 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:12.321 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:12.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 168876 8.58 MB/sec execute 371 sec latency 48.494 ms 2024-09-17T01:10:12.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:12 smithi100 ceph-mon[51701]: pgmap v148: 97 pgs: 97 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.6 MiB/s wr, 317 op/s; 79 KiB/s, 7 objects/s recovering 2024-09-17T01:10:12.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:12.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:13.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 169017 8.56 MB/sec execute 372 sec latency 73.460 ms 2024-09-17T01:10:14.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 169266 8.57 MB/sec execute 373 sec latency 53.049 ms 2024-09-17T01:10:14.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:14 smithi033 ceph-mon[65430]: pgmap v149: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.3 MiB/s wr, 380 op/s; 63 KiB/s, 6 objects/s recovering 2024-09-17T01:10:14.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:14 smithi100 ceph-mon[51701]: pgmap v149: 97 pgs: 97 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.3 MiB/s wr, 380 op/s; 63 KiB/s, 6 objects/s recovering 2024-09-17T01:10:15.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 169744 8.58 MB/sec execute 374 sec latency 40.874 ms 2024-09-17T01:10:16.138 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:16 smithi033 ceph-mon[65430]: pgmap v150: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.1 MiB/s wr, 383 op/s; 57 KiB/s, 5 objects/s recovering 2024-09-17T01:10:16.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 170210 8.59 MB/sec execute 375 sec latency 52.012 ms 2024-09-17T01:10:16.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:16 smithi100 ceph-mon[51701]: pgmap v150: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.1 MiB/s wr, 383 op/s; 57 KiB/s, 5 objects/s recovering 2024-09-17T01:10:17.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 170557 8.60 MB/sec execute 376 sec latency 45.391 ms 2024-09-17T01:10:18.201 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:18 smithi033 ceph-mon[65430]: pgmap v151: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 398 op/s; 43 KiB/s, 5 objects/s recovering 2024-09-17T01:10:18.212 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:18.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 170839 8.60 MB/sec execute 377 sec latency 76.495 ms 2024-09-17T01:10:18.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:18 smithi100 ceph-mon[51701]: pgmap v151: 97 pgs: 97 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 398 op/s; 43 KiB/s, 5 objects/s recovering 2024-09-17T01:10:18.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:19.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 171372 8.62 MB/sec execute 378 sec latency 61.492 ms 2024-09-17T01:10:19.455 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:10:19.966 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:19 smithi033 ceph-mon[65430]: pgmap v152: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.1 MiB/s wr, 440 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T01:10:19.966 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:19 smithi033 ceph-mon[65430]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:19 smithi100 ceph-mon[51701]: pgmap v152: 97 pgs: 97 active+clean; 330 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.1 MiB/s wr, 440 op/s; 0 B/s, 0 objects/s recovering 2024-09-17T01:10:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:19 smithi100 ceph-mon[51701]: from='client.34308 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:20.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 171786 8.64 MB/sec execute 379 sec latency 38.996 ms 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (6m) 22s ago 14m 25.9M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (2m) 22s ago 14m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (2m) 2m ago 14m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (5m) 22s ago 14m 52.9M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (8m) 22s ago 8m 36.5M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (8m) 2m ago 8m 14.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (5m) 22s ago 15m 502M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (4m) 2m ago 14m 440M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (4m) 22s ago 16m 50.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (3m) 2m ago 14m 36.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:10:20.611 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (8m) 22s ago 8m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (8m) 2m ago 8m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (5m) 22s ago 14m 14.8M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (5m) 2m ago 13m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (2m) 22s ago 13m 272M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (80s) 22s ago 12m 197M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (32s) 22s ago 12m 12.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (11m) 22s ago 11m 520M 1937M 17.2.0 e1d6a67b021e 30f2c313c24b 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (11m) 2m ago 11m 449M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (11m) 2m ago 11m 491M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:10:20.612 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (10m) 2m ago 10m 409M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:10:20.613 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (10m) 2m ago 10m 572M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:10:20.613 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (4m) 22s ago 13m 58.3M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:10:20.978 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:20.978 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:20 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:20.978 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:20 smithi033 ceph-mon[65430]: Upgrade: osd.3 is safe to restart 2024-09-17T01:10:20.978 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:20 smithi033 ceph-mon[65430]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:20.978 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:20 smithi033 ceph-mon[65430]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:21.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:21.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:20 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T01:10:21.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:20 smithi100 ceph-mon[51701]: Upgrade: osd.3 is safe to restart 2024-09-17T01:10:21.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:20 smithi100 ceph-mon[51701]: from='client.34312 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:21.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:20 smithi100 ceph-mon[51701]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:21.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 172298 8.62 MB/sec execute 380 sec latency 29.646 ms 2024-09-17T01:10:21.341 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:10:21.341 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:10:21.342 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T01:10:21.343 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T01:10:21.343 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:10:21.343 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:10:21.981 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:10:21.982 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:10:21.982 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:10:21.982 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "9/23 daemons upgraded", 2024-09-17T01:10:21.982 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:10:21.982 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:10:21.982 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:10:21.985 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:21 smithi033 ceph-mon[65430]: pgmap v153: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 427 op/s 2024-09-17T01:10:21.985 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:21 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3419832613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:21 smithi100 ceph-mon[51701]: pgmap v153: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 427 op/s 2024-09-17T01:10:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:21 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3419832613' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:22.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 172660 8.62 MB/sec execute 381 sec latency 37.165 ms 2024-09-17T01:10:22.732 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:10:23.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 173145 8.65 MB/sec execute 382 sec latency 39.111 ms 2024-09-17T01:10:23.510 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:23.510 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: Upgrade: Updating osd.3 2024-09-17T01:10:23.510 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:23.510 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:10:23.510 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:10:23.510 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: Deploying daemon osd.3 on smithi033 2024-09-17T01:10:23.511 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:23 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/532949696' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:10:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: Upgrade: Updating osd.3 2024-09-17T01:10:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:23.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T01:10:23.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:10:23.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: Deploying daemon osd.3 on smithi033 2024-09-17T01:10:23.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:23 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/532949696' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:10:24.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 173811 8.66 MB/sec execute 383 sec latency 30.866 ms 2024-09-17T01:10:24.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:24 smithi033 ceph-mon[65430]: pgmap v154: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.9 MiB/s wr, 574 op/s 2024-09-17T01:10:24.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:24 smithi100 ceph-mon[51701]: pgmap v154: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.9 MiB/s wr, 574 op/s 2024-09-17T01:10:25.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 174230 8.68 MB/sec execute 384 sec latency 25.028 ms 2024-09-17T01:10:26.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 174698 8.69 MB/sec execute 385 sec latency 25.725 ms 2024-09-17T01:10:26.533 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:26 smithi033 ceph-mon[65430]: pgmap v155: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 472 op/s 2024-09-17T01:10:26.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:26 smithi100 ceph-mon[51701]: pgmap v155: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 4.7 MiB/s wr, 472 op/s 2024-09-17T01:10:27.328 INFO:teuthology.orchestra.run.smithi033.stdout: 5 175100 8.71 MB/sec execute 386 sec latency 70.014 ms 2024-09-17T01:10:27.549 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:27.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:28.056 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:27 smithi033 systemd[1]: Stopping Ceph osd.3 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:10:28.310 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:28 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[45132]: 2024-09-17T01:10:28.213+0000 7f86f1b98700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:10:28.310 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:28 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[45132]: 2024-09-17T01:10:28.213+0000 7f86f1b98700 -1 osd.3 105 *** Got signal Terminated *** 2024-09-17T01:10:28.310 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:28 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[45132]: 2024-09-17T01:10:28.213+0000 7f86f1b98700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:10:28.326 INFO:teuthology.orchestra.run.smithi033.stdout: 5 175394 8.71 MB/sec execute 387 sec latency 52.325 ms 2024-09-17T01:10:28.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:28 smithi033 ceph-mon[65430]: pgmap v156: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-09-17T01:10:28.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:28 smithi033 ceph-mon[65430]: osd.3 marked itself down and dead 2024-09-17T01:10:28.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:28 smithi100 ceph-mon[51701]: pgmap v156: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.6 MiB/s wr, 564 op/s 2024-09-17T01:10:28.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:28 smithi100 ceph-mon[51701]: osd.3 marked itself down and dead 2024-09-17T01:10:29.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 175417 8.68 MB/sec execute 388 sec latency 923.828 ms 2024-09-17T01:10:29.363 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:29 smithi033 podman[79032]: 2024-09-17 01:10:29.066334184 +0000 UTC m=+0.992399396 container died 30f2c313c24b5269450e8568bd1a62f934022855713586172be41b4f826e434f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-type=git, maintainer=Guillaume Abrioux , release=754, io.openshift.tags=base centos centos-stream, ceph=True, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T01:10:29.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:29 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:10:29.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:29 smithi033 ceph-mon[65430]: osdmap e106: 8 total, 7 up, 8 in 2024-09-17T01:10:29.677 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:29 smithi033 ceph-mon[65430]: osdmap e107: 8 total, 7 up, 8 in 2024-09-17T01:10:29.677 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:29 smithi033 podman[79032]: 2024-09-17 01:10:29.599648279 +0000 UTC m=+1.525713497 container cleanup 30f2c313c24b5269450e8568bd1a62f934022855713586172be41b4f826e434f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, ceph=True, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, name=centos-stream, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T01:10:29.677 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:29 smithi033 bash[79032]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3 2024-09-17T01:10:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:29 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:10:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:29 smithi100 ceph-mon[51701]: osdmap e106: 8 total, 7 up, 8 in 2024-09-17T01:10:29.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:29 smithi100 ceph-mon[51701]: osdmap e107: 8 total, 7 up, 8 in 2024-09-17T01:10:29.939 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:29 smithi033 podman[79047]: 2024-09-17 01:10:29.885305729 +0000 UTC m=+0.809451251 container remove 30f2c313c24b5269450e8568bd1a62f934022855713586172be41b4f826e434f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, vendor=Red Hat, Inc., version=8, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, ceph=True, distribution-scope=public, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-17T01:10:30.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 175703 8.67 MB/sec execute 389 sec latency 1072.892 ms 2024-09-17T01:10:30.456 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:30 smithi033 ceph-mon[65430]: pgmap v158: 97 pgs: 16 stale+active+clean, 81 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-09-17T01:10:30.456 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:30 smithi033 podman[79133]: 2024-09-17 01:10:30.299272957 +0000 UTC m=+0.022092637 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:10:30.456 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:30 smithi033 podman[79133]: 2024-09-17 01:10:30.411919695 +0000 UTC m=+0.134739364 container create d52471e0fac8cc172e7941d28a3c2ed34d8d6beb218ad4b8548ceb0fc3a733ab (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T01:10:30.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:30 smithi100 ceph-mon[51701]: pgmap v158: 97 pgs: 16 stale+active+clean, 81 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.7 MiB/s wr, 633 op/s 2024-09-17T01:10:31.006 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:30 smithi033 podman[79133]: 2024-09-17 01:10:30.866392967 +0000 UTC m=+0.589212649 container init d52471e0fac8cc172e7941d28a3c2ed34d8d6beb218ad4b8548ceb0fc3a733ab (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:10:31.006 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:30 smithi033 podman[79133]: 2024-09-17 01:10:30.873586681 +0000 UTC m=+0.596406344 container start d52471e0fac8cc172e7941d28a3c2ed34d8d6beb218ad4b8548ceb0fc3a733ab (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:10:31.006 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:31 smithi033 podman[79133]: 2024-09-17 01:10:31.008922632 +0000 UTC m=+0.731742294 container attach d52471e0fac8cc172e7941d28a3c2ed34d8d6beb218ad4b8548ceb0fc3a733ab (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:10:31.288 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:31 smithi033 podman[79133]: 2024-09-17 01:10:31.162118343 +0000 UTC m=+0.884938012 container died d52471e0fac8cc172e7941d28a3c2ed34d8d6beb218ad4b8548ceb0fc3a733ab (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T01:10:31.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 175968 8.65 MB/sec execute 390 sec latency 60.624 ms 2024-09-17T01:10:31.538 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:31 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 74/1698 objects degraded (4.358%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:31.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:31 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 74/1698 objects degraded (4.358%), 7 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:32.061 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:31 smithi033 podman[79156]: 2024-09-17 01:10:31.873787318 +0000 UTC m=+0.702766425 container remove d52471e0fac8cc172e7941d28a3c2ed34d8d6beb218ad4b8548ceb0fc3a733ab (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T01:10:32.061 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:31 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T01:10:32.061 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:31 smithi033 systemd[1]: Stopped Ceph osd.3 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:10:32.061 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:31 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.3.service: Consumed 3min 17.803s CPU time. 2024-09-17T01:10:32.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 176120 8.64 MB/sec execute 391 sec latency 63.518 ms 2024-09-17T01:10:32.338 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 systemd[1]: Starting Ceph osd.3 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:10:32.590 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:32 smithi033 ceph-mon[65430]: pgmap v160: 97 pgs: 2 active+undersized, 5 peering, 12 stale+active+clean, 7 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 538 op/s; 74/1698 objects degraded (4.358%) 2024-09-17T01:10:32.590 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 podman[79250]: 2024-09-17 01:10:32.401404168 +0000 UTC m=+0.022368128 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:10:32.590 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 podman[79250]: 2024-09-17 01:10:32.508442265 +0000 UTC m=+0.129406214 container create f977cb3c97439758dde9d15468cb2d5c89fb8d00090a8f61d3a4806c3012aa7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-17T01:10:32.856 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 podman[79250]: 2024-09-17 01:10:32.803500991 +0000 UTC m=+0.424464943 container init f977cb3c97439758dde9d15468cb2d5c89fb8d00090a8f61d3a4806c3012aa7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T01:10:32.856 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 podman[79250]: 2024-09-17 01:10:32.812687987 +0000 UTC m=+0.433651929 container start f977cb3c97439758dde9d15468cb2d5c89fb8d00090a8f61d3a4806c3012aa7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:10:32.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:32 smithi100 ceph-mon[51701]: pgmap v160: 97 pgs: 2 active+undersized, 5 peering, 12 stale+active+clean, 7 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.4 MiB/s wr, 538 op/s; 74/1698 objects degraded (4.358%) 2024-09-17T01:10:33.113 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 podman[79250]: 2024-09-17 01:10:32.903692922 +0000 UTC m=+0.524656871 container attach f977cb3c97439758dde9d15468cb2d5c89fb8d00090a8f61d3a4806c3012aa7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True) 2024-09-17T01:10:33.113 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:10:33.113 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 bash[79250]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:10:33.113 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:10:33.114 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:32 smithi033 bash[79250]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:10:33.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 176283 8.63 MB/sec execute 392 sec latency 66.806 ms 2024-09-17T01:10:33.641 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:10:33.642 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:10:33.643 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:10:33.643 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate[79265]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 bash[79250]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T01:10:34.166 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:33 smithi033 podman[79250]: 2024-09-17 01:10:33.981328513 +0000 UTC m=+1.602292462 container died f977cb3c97439758dde9d15468cb2d5c89fb8d00090a8f61d3a4806c3012aa7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97) 2024-09-17T01:10:34.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 176597 8.64 MB/sec execute 393 sec latency 52.941 ms 2024-09-17T01:10:34.686 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:34 smithi033 ceph-mon[65430]: pgmap v161: 97 pgs: 12 active+undersized, 5 peering, 24 active+undersized+degraded, 56 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s wr, 674 op/s; 193/1527 objects degraded (12.639%) 2024-09-17T01:10:34.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:34 smithi100 ceph-mon[51701]: pgmap v161: 97 pgs: 12 active+undersized, 5 peering, 24 active+undersized+degraded, 56 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.6 MiB/s wr, 674 op/s; 193/1527 objects degraded (12.639%) 2024-09-17T01:10:35.202 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:34 smithi033 podman[79413]: 2024-09-17 01:10:34.954631143 +0000 UTC m=+0.957545485 container remove f977cb3c97439758dde9d15468cb2d5c89fb8d00090a8f61d3a4806c3012aa7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=) 2024-09-17T01:10:35.202 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:35 smithi033 podman[79470]: 2024-09-17 01:10:35.102823625 +0000 UTC m=+0.022235803 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:10:35.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 177180 8.65 MB/sec execute 394 sec latency 57.947 ms 2024-09-17T01:10:35.454 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:35 smithi033 podman[79470]: 2024-09-17 01:10:35.288772714 +0000 UTC m=+0.208184885 container create 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T01:10:36.221 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:35 smithi033 podman[79470]: 2024-09-17 01:10:35.968558213 +0000 UTC m=+0.887970396 container init 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:10:36.221 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:35 smithi033 podman[79470]: 2024-09-17 01:10:35.979513048 +0000 UTC m=+0.898925213 container start 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux ) 2024-09-17T01:10:36.221 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:36 smithi033 ceph-osd[79488]: -- 172.21.15.33:0/2423407706 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563d11e081e0 con 0x563d10ff5c00 2024-09-17T01:10:36.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 177763 8.68 MB/sec execute 395 sec latency 45.486 ms 2024-09-17T01:10:36.472 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:36 smithi033 bash[79470]: 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 2024-09-17T01:10:36.472 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:36 smithi033 systemd[1]: Started Ceph osd.3 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:10:36.723 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:36 smithi033 ceph-mon[65430]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.2 MiB/s wr, 508 op/s; 203/1527 objects degraded (13.294%); 257 B/s, 0 objects/s recovering 2024-09-17T01:10:36.723 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:36 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:36.723 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:36 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:36.723 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:36 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:10:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:36 smithi100 ceph-mon[51701]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.2 MiB/s wr, 508 op/s; 203/1527 objects degraded (13.294%); 257 B/s, 0 objects/s recovering 2024-09-17T01:10:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:36 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:36 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:36.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:36 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:10:37.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 178210 8.69 MB/sec execute 396 sec latency 126.117 ms 2024-09-17T01:10:37.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:37 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 198/1509 objects degraded (13.121%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:37.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:37 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 198/1509 objects degraded (13.121%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:38.001 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:37 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:10:37.866+0000 7fed6a959640 -1 Falling back to public interface 2024-09-17T01:10:38.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 178508 8.69 MB/sec execute 397 sec latency 82.053 ms 2024-09-17T01:10:38.790 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:38 smithi033 ceph-mon[65430]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.1 MiB/s wr, 411 op/s; 198/1509 objects degraded (13.121%); 240 B/s, 0 objects/s recovering 2024-09-17T01:10:38.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:38 smithi100 ceph-mon[51701]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 326 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.1 MiB/s wr, 411 op/s; 198/1509 objects degraded (13.121%); 240 B/s, 0 objects/s recovering 2024-09-17T01:10:39.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 178781 8.70 MB/sec execute 398 sec latency 52.657 ms 2024-09-17T01:10:40.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 179105 8.69 MB/sec execute 399 sec latency 51.508 ms 2024-09-17T01:10:40.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:40 smithi033 ceph-mon[65430]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 567 op/s; 216/1704 objects degraded (12.676%); 206 B/s, 0 objects/s recovering 2024-09-17T01:10:40.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:40 smithi100 ceph-mon[51701]: pgmap v164: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 5.0 MiB/s wr, 567 op/s; 216/1704 objects degraded (12.676%); 206 B/s, 0 objects/s recovering 2024-09-17T01:10:41.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 179510 8.67 MB/sec execute 400 sec latency 70.967 ms 2024-09-17T01:10:42.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 179714 8.66 MB/sec execute 401 sec latency 48.007 ms 2024-09-17T01:10:42.641 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:42 smithi033 ceph-mon[65430]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.0 MiB/s wr, 357 op/s; 217/1710 objects degraded (12.690%); 177 B/s, 0 objects/s recovering 2024-09-17T01:10:42.641 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:42.641 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:42.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:42 smithi100 ceph-mon[51701]: pgmap v165: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.0 MiB/s wr, 357 op/s; 217/1710 objects degraded (12.690%); 177 B/s, 0 objects/s recovering 2024-09-17T01:10:42.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:42.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:43.327 INFO:teuthology.orchestra.run.smithi033.stdout: 5 180100 8.69 MB/sec execute 402 sec latency 43.118 ms 2024-09-17T01:10:43.648 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:43 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 202/1584 objects degraded (12.753%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:43.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:43 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 202/1584 objects degraded (12.753%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:44.328 INFO:teuthology.orchestra.run.smithi033.stdout: 5 180566 8.69 MB/sec execute 403 sec latency 36.627 ms 2024-09-17T01:10:44.659 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:44 smithi033 ceph-mon[65430]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.3 MiB/s wr, 546 op/s; 202/1584 objects degraded (12.753%); 171 B/s, 0 objects/s recovering 2024-09-17T01:10:44.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:44 smithi100 ceph-mon[51701]: pgmap v166: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 5.3 MiB/s wr, 546 op/s; 202/1584 objects degraded (12.753%); 171 B/s, 0 objects/s recovering 2024-09-17T01:10:45.328 INFO:teuthology.orchestra.run.smithi033.stdout: 5 181012 8.69 MB/sec execute 404 sec latency 37.787 ms 2024-09-17T01:10:46.328 INFO:teuthology.orchestra.run.smithi033.stdout: 5 181310 8.70 MB/sec execute 405 sec latency 53.857 ms 2024-09-17T01:10:46.445 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:46 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:10:46.375+0000 7fed6a959640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T01:10:46.697 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:46 smithi033 ceph-mon[65430]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.6 MiB/s wr, 472 op/s; 203/1587 objects degraded (12.791%); 171 B/s, 0 objects/s recovering 2024-09-17T01:10:46.698 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:46 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:46.698 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:46 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:46.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:46 smithi100 ceph-mon[51701]: pgmap v167: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.6 MiB/s wr, 472 op/s; 203/1587 objects degraded (12.791%); 171 B/s, 0 objects/s recovering 2024-09-17T01:10:46.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:46 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:46.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:46 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:47.330 INFO:teuthology.orchestra.run.smithi033.stdout: 5 181679 8.70 MB/sec execute 406 sec latency 61.836 ms 2024-09-17T01:10:47.710 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:47 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:10:47.682+0000 7fed6a959640 -1 osd.3 105 log_to_monitors true 2024-09-17T01:10:48.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 182034 8.71 MB/sec execute 407 sec latency 60.812 ms 2024-09-17T01:10:48.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:48 smithi033 ceph-mon[65430]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 491 op/s; 201/1605 objects degraded (12.523%) 2024-09-17T01:10:48.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:48.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:48.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:48 smithi033 ceph-mon[65430]: from='osd.3 [v2:172.21.15.33:6826/2127824973,v1:172.21.15.33:6827/2127824973]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:10:48.497 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:48 smithi033 ceph-mon[65430]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:10:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:48 smithi100 ceph-mon[51701]: pgmap v168: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.0 MiB/s wr, 491 op/s; 201/1605 objects degraded (12.523%) 2024-09-17T01:10:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:48.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:48 smithi100 ceph-mon[51701]: from='osd.3 [v2:172.21.15.33:6826/2127824973,v1:172.21.15.33:6827/2127824973]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:10:48.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:48 smithi100 ceph-mon[51701]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T01:10:48.760 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:10:48 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:10:48.705+0000 7fed62704640 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:10:49.334 INFO:teuthology.orchestra.run.smithi033.stdout: 5 182276 8.71 MB/sec execute 408 sec latency 46.402 ms 2024-09-17T01:10:49.783 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:49 smithi033 ceph-mon[65430]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:10:49.783 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:49 smithi033 ceph-mon[65430]: from='osd.3 [v2:172.21.15.33:6826/2127824973,v1:172.21.15.33:6827/2127824973]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:49.784 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:49 smithi033 ceph-mon[65430]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T01:10:49.784 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:49 smithi033 ceph-mon[65430]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:49.784 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:49 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 201/1605 objects degraded (12.523%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:49 smithi100 ceph-mon[51701]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T01:10:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:49 smithi100 ceph-mon[51701]: from='osd.3 [v2:172.21.15.33:6826/2127824973,v1:172.21.15.33:6827/2127824973]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:49 smithi100 ceph-mon[51701]: osdmap e108: 8 total, 7 up, 8 in 2024-09-17T01:10:49.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:49 smithi100 ceph-mon[51701]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-17T01:10:49.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:49 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 201/1605 objects degraded (12.523%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:50.334 INFO:teuthology.orchestra.run.smithi033.stdout: 5 182448 8.70 MB/sec execute 409 sec latency 675.223 ms 2024-09-17T01:10:50.827 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:50 smithi033 ceph-mon[65430]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 466 op/s; 208/1686 objects degraded (12.337%) 2024-09-17T01:10:50.827 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:50 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:10:50.827 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:50 smithi033 ceph-mon[65430]: osd.3 [v2:172.21.15.33:6826/2127824973,v1:172.21.15.33:6827/2127824973] boot 2024-09-17T01:10:50.827 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:50 smithi033 ceph-mon[65430]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T01:10:50.827 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:50 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:10:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:50 smithi100 ceph-mon[51701]: pgmap v170: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 334 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 466 op/s; 208/1686 objects degraded (12.337%) 2024-09-17T01:10:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:50 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:10:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:50 smithi100 ceph-mon[51701]: osd.3 [v2:172.21.15.33:6826/2127824973,v1:172.21.15.33:6827/2127824973] boot 2024-09-17T01:10:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:50 smithi100 ceph-mon[51701]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T01:10:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:50 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T01:10:51.334 INFO:teuthology.orchestra.run.smithi033.stdout: 5 182652 8.69 MB/sec execute 410 sec latency 1675.255 ms 2024-09-17T01:10:51.915 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:51 smithi033 ceph-mon[65430]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T01:10:51.915 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:51 smithi033 ceph-mon[65430]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T01:10:51.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:51 smithi100 ceph-mon[51701]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T01:10:51.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:51 smithi100 ceph-mon[51701]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T01:10:52.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 182876 8.67 MB/sec execute 411 sec latency 2035.440 ms 2024-09-17T01:10:52.679 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:52 smithi033 ceph-mon[65430]: pgmap v173: 97 pgs: 1 unknown, 5 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 446 op/s; 173/1692 objects degraded (10.225%) 2024-09-17T01:10:52.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:52 smithi100 ceph-mon[51701]: pgmap v173: 97 pgs: 1 unknown, 5 peering, 12 active+undersized, 23 active+undersized+degraded, 56 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 446 op/s; 173/1692 objects degraded (10.225%) 2024-09-17T01:10:53.211 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:10:53.334 INFO:teuthology.orchestra.run.smithi033.stdout: 5 183213 8.67 MB/sec execute 412 sec latency 658.552 ms 2024-09-17T01:10:53.948 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:10:53.948 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.948 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.948 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:10:53.948 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:53.949 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:10:53.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T01:10:53.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:53.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (6m) 7s ago 15m 26.4M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (3m) 7s ago 15m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (3m) 3m ago 14m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (6m) 7s ago 15m 53.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (9m) 7s ago 9m 36.5M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:10:54.223 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (9m) 3m ago 9m 14.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (5m) 7s ago 16m 504M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (4m) 3m ago 14m 440M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (4m) 7s ago 16m 50.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (4m) 3m ago 14m 36.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (9m) 7s ago 9m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (9m) 3m ago 8m 85.9M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (6m) 7s ago 15m 15.0M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (5m) 3m ago 14m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (2m) 7s ago 13m 331M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (113s) 7s ago 13m 269M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (66s) 7s ago 12m 188M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (19s) 7s ago 12m 14.5M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:10:54.224 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (12m) 3m ago 12m 449M 2703M 17.2.0 e1d6a67b021e 1fa0fc4b0282 2024-09-17T01:10:54.225 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (11m) 3m ago 11m 491M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:10:54.225 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (11m) 3m ago 11m 409M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:10:54.225 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (10m) 3m ago 10m 572M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:10:54.225 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (4m) 7s ago 14m 62.7M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:10:54.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 183604 8.66 MB/sec execute 413 sec latency 1053.456 ms 2024-09-17T01:10:54.838 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:10:54.838 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:10:54.838 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T01:10:54.839 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:10:54.840 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:10:54.840 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:10:54.840 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: pgmap v176: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 13 active+undersized+degraded, 66 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 697 op/s; 136/1608 objects degraded (8.458%); 1024 KiB/s, 11 objects/s recovering 2024-09-17T01:10:54.840 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:54.840 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:10:54.840 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: Upgrade: 1 pgs have unknown state; cannot draw any conclusions 2024-09-17T01:10:54.840 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:10:54.840 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: from='client.44241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:54.841 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:54 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 136/1608 objects degraded (8.458%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:54.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: pgmap v176: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 6 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized, 13 active+undersized+degraded, 66 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.6 MiB/s wr, 697 op/s; 136/1608 objects degraded (8.458%); 1024 KiB/s, 11 objects/s recovering 2024-09-17T01:10:54.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: from='client.34336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:54.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:10:54.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: Upgrade: 1 pgs have unknown state; cannot draw any conclusions 2024-09-17T01:10:54.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T01:10:54.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: from='client.44241 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:54.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:54 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 136/1608 objects degraded (8.458%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:55.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 183953 8.66 MB/sec execute 414 sec latency 919.160 ms 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:10:55.399 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:10:55.400 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:10:55.400 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:10:55.400 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "10/23 daemons upgraded", 2024-09-17T01:10:55.400 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:10:55.400 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:10:55.400 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:10:55.967 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN Degraded data redundancy: 64/1599 objects degraded (4.003%), 4 pgs degraded 2024-09-17T01:10:55.967 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 64/1599 objects degraded (4.003%), 4 pgs degraded 2024-09-17T01:10:55.967 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.2 is active+recovery_wait+undersized+degraded+remapped, acting [5,6] 2024-09-17T01:10:55.967 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-17T01:10:55.967 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.c is active+recovering+undersized+degraded+remapped, acting [5,6] 2024-09-17T01:10:55.967 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.15 is active+recovery_wait+undersized+degraded+remapped, acting [7,4] 2024-09-17T01:10:55.968 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:55 smithi033 ceph-mon[65430]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:55.968 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:55 smithi033 ceph-mon[65430]: osdmap e114: 8 total, 8 up, 8 in 2024-09-17T01:10:55.968 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:55 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3267440958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:55.968 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:55 smithi033 ceph-mon[65430]: osdmap e115: 8 total, 8 up, 8 in 2024-09-17T01:10:55.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:55 smithi100 ceph-mon[51701]: from='client.34344 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:55.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:55 smithi100 ceph-mon[51701]: osdmap e114: 8 total, 8 up, 8 in 2024-09-17T01:10:55.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:55 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3267440958' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:10:55.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:55 smithi100 ceph-mon[51701]: osdmap e115: 8 total, 8 up, 8 in 2024-09-17T01:10:56.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 184210 8.66 MB/sec execute 415 sec latency 621.994 ms 2024-09-17T01:10:56.735 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:56 smithi033 ceph-mon[65430]: pgmap v179: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 8 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 82 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.7 MiB/s wr, 463 op/s; 64/1599 objects degraded (4.003%); 2.4 MiB/s, 33 objects/s recovering 2024-09-17T01:10:56.735 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:56 smithi033 ceph-mon[65430]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:56.735 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:56 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/680378223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:10:56.735 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:56 smithi033 ceph-mon[65430]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T01:10:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:56 smithi100 ceph-mon[51701]: pgmap v179: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 1 unknown, 8 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 82 active+clean; 329 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.7 MiB/s wr, 463 op/s; 64/1599 objects degraded (4.003%); 2.4 MiB/s, 33 objects/s recovering 2024-09-17T01:10:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:56 smithi100 ceph-mon[51701]: from='client.34352 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:10:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:56 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/680378223' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:10:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:56 smithi100 ceph-mon[51701]: osdmap e116: 8 total, 8 up, 8 in 2024-09-17T01:10:57.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 184524 8.67 MB/sec execute 416 sec latency 1007.907 ms 2024-09-17T01:10:58.275 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:58 smithi033 ceph-mon[65430]: pgmap v182: 97 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 88 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.2 MiB/s wr, 93 op/s; 78/1578 objects degraded (4.943%); 2.4 MiB/s, 42 objects/s recovering 2024-09-17T01:10:58.275 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:58.275 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:58.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 184863 8.68 MB/sec execute 417 sec latency 63.517 ms 2024-09-17T01:10:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:58 smithi100 ceph-mon[51701]: pgmap v182: 97 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 88 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 1.2 MiB/s wr, 93 op/s; 78/1578 objects degraded (4.943%); 2.4 MiB/s, 42 objects/s recovering 2024-09-17T01:10:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:10:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:10:59.292 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:10:59 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 78/1578 objects degraded (4.943%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:10:59.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 185034 8.68 MB/sec execute 418 sec latency 68.861 ms 2024-09-17T01:10:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:10:59 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 78/1578 objects degraded (4.943%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:00.096 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:00 smithi033 ceph-mon[65430]: pgmap v183: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.5 MiB/s wr, 455 op/s; 31/1635 objects degraded (1.896%); 1.8 MiB/s, 24 objects/s recovering 2024-09-17T01:11:00.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 185276 8.67 MB/sec execute 419 sec latency 99.993 ms 2024-09-17T01:11:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:00 smithi100 ceph-mon[51701]: pgmap v183: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 4.5 MiB/s wr, 455 op/s; 31/1635 objects degraded (1.896%); 1.8 MiB/s, 24 objects/s recovering 2024-09-17T01:11:01.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:01 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1635 objects degraded (1.896%), 2 pgs degraded) 2024-09-17T01:11:01.124 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:01 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:11:01.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 185450 8.67 MB/sec execute 420 sec latency 74.166 ms 2024-09-17T01:11:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:01 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/1635 objects degraded (1.896%), 2 pgs degraded) 2024-09-17T01:11:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:01 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:11:02.192 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:02 smithi033 ceph-mon[65430]: pgmap v184: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 405 op/s; 794 KiB/s, 26 objects/s recovering 2024-09-17T01:11:02.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 185658 8.66 MB/sec execute 421 sec latency 73.222 ms 2024-09-17T01:11:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:02 smithi100 ceph-mon[51701]: pgmap v184: 97 pgs: 97 active+clean; 335 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 405 op/s; 794 KiB/s, 26 objects/s recovering 2024-09-17T01:11:03.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 185901 8.65 MB/sec execute 422 sec latency 43.462 ms 2024-09-17T01:11:04.224 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:04 smithi033 ceph-mon[65430]: pgmap v185: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.0 MiB/s wr, 474 op/s; 625 KiB/s, 20 objects/s recovering 2024-09-17T01:11:04.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 186191 8.64 MB/sec execute 423 sec latency 40.355 ms 2024-09-17T01:11:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:04 smithi100 ceph-mon[51701]: pgmap v185: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 5.0 MiB/s wr, 474 op/s; 625 KiB/s, 20 objects/s recovering 2024-09-17T01:11:05.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 186500 8.63 MB/sec execute 424 sec latency 54.013 ms 2024-09-17T01:11:06.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:06 smithi033 ceph-mon[65430]: pgmap v186: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 442 op/s; 519 KiB/s, 11 objects/s recovering 2024-09-17T01:11:06.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 186796 8.63 MB/sec execute 425 sec latency 39.371 ms 2024-09-17T01:11:06.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:06 smithi100 ceph-mon[51701]: pgmap v186: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.5 MiB/s wr, 442 op/s; 519 KiB/s, 11 objects/s recovering 2024-09-17T01:11:07.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 187331 8.65 MB/sec execute 426 sec latency 41.542 ms 2024-09-17T01:11:08.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:08 smithi033 ceph-mon[65430]: pgmap v187: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 403 op/s; 70 KiB/s, 8 objects/s recovering 2024-09-17T01:11:08.271 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:08 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:08.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 187872 8.65 MB/sec execute 427 sec latency 38.228 ms 2024-09-17T01:11:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:08 smithi100 ceph-mon[51701]: pgmap v187: 97 pgs: 97 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.2 MiB/s wr, 403 op/s; 70 KiB/s, 8 objects/s recovering 2024-09-17T01:11:08.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:08 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:09.301 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:09 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:11:09.301 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:09 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:11:09.301 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:09 smithi033 ceph-mon[65430]: Upgrade: osd.4 is safe to restart 2024-09-17T01:11:09.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 188258 8.67 MB/sec execute 428 sec latency 48.418 ms 2024-09-17T01:11:09.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:09 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:11:09.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:09 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T01:11:09.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:09 smithi100 ceph-mon[51701]: Upgrade: osd.4 is safe to restart 2024-09-17T01:11:10.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 188598 8.68 MB/sec execute 429 sec latency 94.166 ms 2024-09-17T01:11:10.344 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:10 smithi033 ceph-mon[65430]: pgmap v188: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.6 MiB/s wr, 408 op/s; 60 KiB/s, 7 objects/s recovering 2024-09-17T01:11:10.344 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:10.344 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:11:10.344 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:11:10.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:10 smithi100 ceph-mon[51701]: pgmap v188: 97 pgs: 97 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.8 KiB/s rd, 4.6 MiB/s wr, 408 op/s; 60 KiB/s, 7 objects/s recovering 2024-09-17T01:11:10.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:10.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T01:11:10.392 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:11:11.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 188912 8.68 MB/sec execute 430 sec latency 52.429 ms 2024-09-17T01:11:11.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:11 smithi033 ceph-mon[65430]: Upgrade: Updating osd.4 2024-09-17T01:11:11.360 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:11 smithi033 ceph-mon[65430]: Deploying daemon osd.4 on smithi100 2024-09-17T01:11:11.434 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:11 smithi100 ceph-mon[51701]: Upgrade: Updating osd.4 2024-09-17T01:11:11.435 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:11 smithi100 ceph-mon[51701]: Deploying daemon osd.4 on smithi100 2024-09-17T01:11:12.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 189176 8.68 MB/sec execute 431 sec latency 58.861 ms 2024-09-17T01:11:12.372 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:12 smithi033 ceph-mon[65430]: pgmap v189: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.9 MiB/s wr, 405 op/s; 60 KiB/s, 3 objects/s recovering 2024-09-17T01:11:12.372 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:12.372 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:12.465 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:12 smithi100 ceph-mon[51701]: pgmap v189: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.9 MiB/s wr, 405 op/s; 60 KiB/s, 3 objects/s recovering 2024-09-17T01:11:12.465 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:12.465 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:13.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 189400 8.67 MB/sec execute 432 sec latency 75.580 ms 2024-09-17T01:11:13.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:13 smithi100 systemd[1]: Stopping Ceph osd.4 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:11:13.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:13 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[30053]: 2024-09-17T01:11:13.571+0000 7fe1bbd4b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:11:13.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:13 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[30053]: 2024-09-17T01:11:13.571+0000 7fe1bbd4b700 -1 osd.4 116 *** Got signal Terminated *** 2024-09-17T01:11:13.978 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:13 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[30053]: 2024-09-17T01:11:13.571+0000 7fe1bbd4b700 -1 osd.4 116 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:11:14.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 189634 8.66 MB/sec execute 433 sec latency 142.310 ms 2024-09-17T01:11:14.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:14 smithi033 ceph-mon[65430]: pgmap v190: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.1 MiB/s wr, 527 op/s 2024-09-17T01:11:14.438 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:14 smithi033 ceph-mon[65430]: osd.4 marked itself down and dead 2024-09-17T01:11:14.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:14 smithi100 ceph-mon[51701]: pgmap v190: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 5.1 MiB/s wr, 527 op/s 2024-09-17T01:11:14.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:14 smithi100 ceph-mon[51701]: osd.4 marked itself down and dead 2024-09-17T01:11:15.061 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:14 smithi100 podman[56126]: 2024-09-17 01:11:14.789333875 +0000 UTC m=+1.363978161 container died 1fa0fc4b0282cc33fbf2de2f3124147bbd79d36cedffd378abc2212970916182 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , RELEASE=HEAD, version=8, io.buildah.version=1.19.8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870) 2024-09-17T01:11:15.232 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:15 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:11:15.233 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:15 smithi033 ceph-mon[65430]: osdmap e117: 8 total, 7 up, 8 in 2024-09-17T01:11:15.233 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:15 smithi033 ceph-mon[65430]: osdmap e118: 8 total, 7 up, 8 in 2024-09-17T01:11:15.336 INFO:teuthology.orchestra.run.smithi033.stdout: 5 189750 8.64 MB/sec execute 434 sec latency 1020.754 ms 2024-09-17T01:11:15.446 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:15 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:11:15.446 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:15 smithi100 ceph-mon[51701]: osdmap e117: 8 total, 7 up, 8 in 2024-09-17T01:11:15.446 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:15 smithi100 ceph-mon[51701]: osdmap e118: 8 total, 7 up, 8 in 2024-09-17T01:11:15.446 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:15 smithi100 podman[56126]: 2024-09-17 01:11:15.14669061 +0000 UTC m=+1.721334892 container cleanup 1fa0fc4b0282cc33fbf2de2f3124147bbd79d36cedffd378abc2212970916182 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, ceph=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, GIT_CLEAN=True, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64) 2024-09-17T01:11:15.446 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:15 smithi100 bash[56126]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4 2024-09-17T01:11:15.727 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:15 smithi100 podman[56142]: 2024-09-17 01:11:15.44863848 +0000 UTC m=+0.658481635 container remove 1fa0fc4b0282cc33fbf2de2f3124147bbd79d36cedffd378abc2212970916182 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, GIT_BRANCH=HEAD, release=754, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD) 2024-09-17T01:11:16.204 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:15 smithi100 podman[56226]: 2024-09-17 01:11:15.701296255 +0000 UTC m=+0.017721958 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:11:16.204 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:15 smithi100 podman[56226]: 2024-09-17 01:11:15.83331606 +0000 UTC m=+0.149741740 container create f3ea393deaa0f417e6bd9675f495bc8048f32f180af2bc6a22b770ca7d87e21f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:11:16.266 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:16 smithi033 ceph-mon[65430]: pgmap v192: 97 pgs: 14 stale+active+clean, 83 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-09-17T01:11:16.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 190042 8.63 MB/sec execute 435 sec latency 70.889 ms 2024-09-17T01:11:16.458 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:16 smithi100 ceph-mon[51701]: pgmap v192: 97 pgs: 14 stale+active+clean, 83 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 482 op/s 2024-09-17T01:11:16.459 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:16 smithi100 podman[56226]: 2024-09-17 01:11:16.36163358 +0000 UTC m=+0.678059273 container init f3ea393deaa0f417e6bd9675f495bc8048f32f180af2bc6a22b770ca7d87e21f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:11:16.459 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:16 smithi100 podman[56226]: 2024-09-17 01:11:16.365961714 +0000 UTC m=+0.682387402 container start f3ea393deaa0f417e6bd9675f495bc8048f32f180af2bc6a22b770ca7d87e21f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-17T01:11:16.459 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:16 smithi100 podman[56226]: 2024-09-17 01:11:16.461601848 +0000 UTC m=+0.778027533 container attach f3ea393deaa0f417e6bd9675f495bc8048f32f180af2bc6a22b770ca7d87e21f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:11:16.727 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:16 smithi100 podman[56226]: 2024-09-17 01:11:16.580778476 +0000 UTC m=+0.897204156 container died f3ea393deaa0f417e6bd9675f495bc8048f32f180af2bc6a22b770ca7d87e21f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T01:11:17.299 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:17 smithi033 ceph-mon[65430]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T01:11:17.299 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:17 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 33/1680 objects degraded (1.964%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:17.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 190389 8.63 MB/sec execute 436 sec latency 51.545 ms 2024-09-17T01:11:17.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:17 smithi100 ceph-mon[51701]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T01:11:17.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:17 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 33/1680 objects degraded (1.964%), 4 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:17.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 podman[56249]: 2024-09-17 01:11:17.217739388 +0000 UTC m=+0.630307112 container remove f3ea393deaa0f417e6bd9675f495bc8048f32f180af2bc6a22b770ca7d87e21f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T01:11:17.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T01:11:17.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 systemd[1]: Stopped Ceph osd.4 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:11:17.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 289ms CPU time. 2024-09-17T01:11:17.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 systemd[1]: Starting Ceph osd.4 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:11:17.945 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 podman[56344]: 2024-09-17 01:11:17.57826309 +0000 UTC m=+0.017524089 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:11:17.945 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 podman[56344]: 2024-09-17 01:11:17.695182789 +0000 UTC m=+0.134443773 container create c827897d8ec5f0100fe02d892ae30a53add7dcb95ffec3c5f07a1d102a44d9a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97) 2024-09-17T01:11:17.945 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 podman[56344]: 2024-09-17 01:11:17.948287815 +0000 UTC m=+0.387548810 container init c827897d8ec5f0100fe02d892ae30a53add7dcb95ffec3c5f07a1d102a44d9a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T01:11:18.227 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:17 smithi100 podman[56344]: 2024-09-17 01:11:17.951750427 +0000 UTC m=+0.391011412 container start c827897d8ec5f0100fe02d892ae30a53add7dcb95ffec3c5f07a1d102a44d9a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-17T01:11:18.227 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:18 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:18.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:18 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:18.307 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:18 smithi033 ceph-mon[65430]: pgmap v194: 97 pgs: 2 active+undersized, 8 peering, 10 stale+active+clean, 4 active+undersized+degraded, 73 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.8 MiB/s wr, 516 op/s; 33/1680 objects degraded (1.964%) 2024-09-17T01:11:18.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 190784 8.64 MB/sec execute 437 sec latency 48.240 ms 2024-09-17T01:11:18.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:18 smithi100 ceph-mon[51701]: pgmap v194: 97 pgs: 2 active+undersized, 8 peering, 10 stale+active+clean, 4 active+undersized+degraded, 73 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.8 MiB/s wr, 516 op/s; 33/1680 objects degraded (1.964%) 2024-09-17T01:11:18.728 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:18 smithi100 bash[56344]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:18.728 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:18 smithi100 bash[56344]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:18.728 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:18 smithi100 podman[56344]: 2024-09-17 01:11:18.255339197 +0000 UTC m=+0.694600189 container attach c827897d8ec5f0100fe02d892ae30a53add7dcb95ffec3c5f07a1d102a44d9a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T01:11:19.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 191336 8.65 MB/sec execute 438 sec latency 36.450 ms 2024-09-17T01:11:19.857 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:11:19.857 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:11:19.857 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:11:19.857 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T01:11:19.858 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:11:19.858 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T01:11:20.227 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate[56359]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 bash[56344]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T01:11:20.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:19 smithi100 podman[56344]: 2024-09-17 01:11:19.891657192 +0000 UTC m=+2.330918189 container died c827897d8ec5f0100fe02d892ae30a53add7dcb95ffec3c5f07a1d102a44d9a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-17T01:11:20.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 191660 8.65 MB/sec execute 439 sec latency 64.512 ms 2024-09-17T01:11:20.346 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:20 smithi033 ceph-mon[65430]: pgmap v195: 97 pgs: 10 active+undersized, 8 peering, 2 stale+active+clean, 20 active+undersized+degraded, 57 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.7 MiB/s wr, 414 op/s; 164/1605 objects degraded (10.218%) 2024-09-17T01:11:20.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:20 smithi100 ceph-mon[51701]: pgmap v195: 97 pgs: 10 active+undersized, 8 peering, 2 stale+active+clean, 20 active+undersized+degraded, 57 active+clean; 330 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.7 MiB/s wr, 414 op/s; 164/1605 objects degraded (10.218%) 2024-09-17T01:11:21.186 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:20 smithi100 podman[56502]: 2024-09-17 01:11:20.829940594 +0000 UTC m=+0.930202778 container remove c827897d8ec5f0100fe02d892ae30a53add7dcb95ffec3c5f07a1d102a44d9a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True) 2024-09-17T01:11:21.187 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 podman[56594]: 2024-09-17 01:11:20.943810082 +0000 UTC m=+0.016953899 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:11:21.187 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 podman[56594]: 2024-09-17 01:11:21.064531042 +0000 UTC m=+0.137674845 container create c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T01:11:21.336 INFO:teuthology.orchestra.run.smithi033.stdout: 5 192235 8.68 MB/sec execute 440 sec latency 42.097 ms 2024-09-17T01:11:21.357 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:21 smithi033 ceph-mon[65430]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T01:11:21.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:21 smithi100 ceph-mon[51701]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T01:11:21.727 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 podman[56594]: 2024-09-17 01:11:21.325636128 +0000 UTC m=+0.398779951 container init c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-17T01:11:21.727 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 podman[56594]: 2024-09-17 01:11:21.329093931 +0000 UTC m=+0.402237752 container start c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-09-17T01:11:21.728 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 ceph-osd[56613]: -- 172.21.15.100:0/3806671523 <== mon.1 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559ed6fb61e0 con 0x559ed61a7c00 2024-09-17T01:11:21.728 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 bash[56594]: c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 2024-09-17T01:11:21.728 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:21 smithi100 systemd[1]: Started Ceph osd.4 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:11:22.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 192688 8.70 MB/sec execute 441 sec latency 75.957 ms 2024-09-17T01:11:22.633 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:22 smithi033 ceph-mon[65430]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.1 MiB/s wr, 253 op/s; 238/1614 objects degraded (14.746%) 2024-09-17T01:11:22.633 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:22.633 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:22.633 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:11:22.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:22 smithi100 ceph-mon[51701]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.1 MiB/s wr, 253 op/s; 238/1614 objects degraded (14.746%) 2024-09-17T01:11:22.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:22.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:22.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:11:23.227 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:23 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:11:23.202+0000 7f9756ed8640 -1 Falling back to public interface 2024-09-17T01:11:23.335 INFO:teuthology.orchestra.run.smithi033.stdout: 5 192848 8.69 MB/sec execute 442 sec latency 60.954 ms 2024-09-17T01:11:23.647 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:23 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 247/1671 objects degraded (14.782%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:23.788 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:23 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 247/1671 objects degraded (14.782%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:24.336 INFO:teuthology.orchestra.run.smithi033.stdout: 5 193027 8.68 MB/sec execute 443 sec latency 70.983 ms 2024-09-17T01:11:24.695 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:24 smithi033 ceph-mon[65430]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 431 op/s; 247/1671 objects degraded (14.782%) 2024-09-17T01:11:24.845 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:24 smithi100 ceph-mon[51701]: pgmap v197: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 431 op/s; 247/1671 objects degraded (14.782%) 2024-09-17T01:11:25.336 INFO:teuthology.orchestra.run.smithi033.stdout: 5 193351 8.66 MB/sec execute 444 sec latency 85.153 ms 2024-09-17T01:11:26.234 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:26 smithi033 ceph-mon[65430]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 405 op/s; 252/1713 objects degraded (14.711%) 2024-09-17T01:11:26.234 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.234 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.234 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.234 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 193577 8.65 MB/sec execute 445 sec latency 61.270 ms 2024-09-17T01:11:26.464 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:11:26.472 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:26 smithi100 ceph-mon[51701]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.5 MiB/s wr, 405 op/s; 252/1713 objects degraded (14.711%) 2024-09-17T01:11:26.472 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.472 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.472 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:26.472 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:27.260 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:27 smithi033 ceph-mon[65430]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:27.260 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:27.260 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:27.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 193786 8.64 MB/sec execute 446 sec latency 51.837 ms 2024-09-17T01:11:27.438 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:11:27.438 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (7m) 41s ago 15m 26.4M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:11:27.438 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (3m) 41s ago 15m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:11:27.438 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (3m) 2s ago 15m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:11:27.438 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (7m) 41s ago 15m 53.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:11:27.438 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (10m) 41s ago 9m 36.5M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (9m) 2s ago 9m 14.8M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (6m) 41s ago 17m 504M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (5m) 2s ago 15m 443M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (5m) 41s ago 17m 50.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (4m) 2s ago 15m 41.2M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (9m) 41s ago 9m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (9m) 2s ago 9m 88.5M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (6m) 41s ago 15m 15.0M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (6m) 2s ago 14m 16.6M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (3m) 41s ago 14m 331M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (2m) 41s ago 13m 269M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (99s) 41s ago 13m 188M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (52s) 41s ago 13m 14.5M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (6s) 2s ago 12m 14.2M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:11:27.439 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (12m) 2s ago 12m 459M 2703M 17.2.0 e1d6a67b021e 83cd9cb1d6b8 2024-09-17T01:11:27.440 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (11m) 2s ago 11m 353M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:11:27.440 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (11m) 2s ago 11m 558M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:11:27.440 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (5m) 41s ago 14m 62.7M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:11:27.476 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:27 smithi100 ceph-mon[51701]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:27.476 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:27.476 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:28.011 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:11:28.012 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:11:28.013 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:11:28.013 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:11:28.013 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T01:11:28.013 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:11:28.013 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:11:28.013 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:11:28.298 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:28 smithi033 ceph-mon[65430]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:28.298 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:28 smithi033 ceph-mon[65430]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 436 op/s; 246/1683 objects degraded (14.617%) 2024-09-17T01:11:28.298 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:28 smithi033 ceph-mon[65430]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:28.298 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:28 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/4080647321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:28.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 194038 8.64 MB/sec execute 447 sec latency 71.113 ms 2024-09-17T01:11:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:28 smithi100 ceph-mon[51701]: from='client.44263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:28 smithi100 ceph-mon[51701]: pgmap v199: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.4 MiB/s wr, 436 op/s; 246/1683 objects degraded (14.617%) 2024-09-17T01:11:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:28 smithi100 ceph-mon[51701]: from='client.34370 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:28.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:28 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/4080647321' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:28.628 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:11:28.628 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:11:28.628 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:11:28.628 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:11:28.628 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "11/23 daemons upgraded", 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:11:28.629 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 229/1527 objects degraded (14.997%), 28 pgs degraded 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout: osd.4 (root=default,host=smithi100) is down 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 229/1527 objects degraded (14.997%), 28 pgs degraded 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-17T01:11:29.271 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.f is active+undersized+degraded, acting [0,2] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-17T01:11:29.272 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-17T01:11:29.273 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-17T01:11:29.343 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:29 smithi033 ceph-mon[65430]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:29.343 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:29 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 246/1683 objects degraded (14.617%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:29.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 194310 8.64 MB/sec execute 448 sec latency 57.562 ms 2024-09-17T01:11:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:29 smithi100 ceph-mon[51701]: from='client.44275 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:11:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:29 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 246/1683 objects degraded (14.617%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:29.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:29 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:11:29.492+0000 7f9756ed8640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T01:11:30.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 194616 8.63 MB/sec execute 449 sec latency 69.820 ms 2024-09-17T01:11:30.405 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.3 MiB/s wr, 563 op/s; 229/1527 objects degraded (14.997%) 2024-09-17T01:11:30.405 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3981576277' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:30.406 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:30.590 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: pgmap v200: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.3 MiB/s wr, 563 op/s; 229/1527 objects degraded (14.997%) 2024-09-17T01:11:30.590 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3981576277' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:11:30.590 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.590 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.590 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:11:30.590 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:11:30.591 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:30.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:30 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:11:30.592+0000 7f9756ed8640 -1 osd.4 116 log_to_monitors true 2024-09-17T01:11:31.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 194974 8.64 MB/sec execute 450 sec latency 93.343 ms 2024-09-17T01:11:31.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:31 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:31.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:31 smithi033 ceph-mon[65430]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T01:11:31.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:31 smithi033 ceph-mon[65430]: from='osd.4 [v2:172.21.15.100:6800/3545662876,v1:172.21.15.100:6801/3545662876]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:11:31.460 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:31 smithi033 ceph-mon[65430]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:11:31.721 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:31 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:31.721 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:31 smithi100 ceph-mon[51701]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T01:11:31.721 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:31 smithi100 ceph-mon[51701]: from='osd.4 [v2:172.21.15.100:6800/3545662876,v1:172.21.15.100:6801/3545662876]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:11:31.721 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:31 smithi100 ceph-mon[51701]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T01:11:31.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:11:31 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:11:31.723+0000 7f974ac7b640 -1 osd.4 116 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:11:32.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195153 8.63 MB/sec execute 451 sec latency 157.451 ms 2024-09-17T01:11:32.413 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:32 smithi033 ceph-mon[65430]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.6 MiB/s wr, 473 op/s; 228/1521 objects degraded (14.990%) 2024-09-17T01:11:32.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:32 smithi033 ceph-mon[65430]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:11:32.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:32 smithi033 ceph-mon[65430]: osdmap e119: 8 total, 7 up, 8 in 2024-09-17T01:11:32.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:32 smithi033 ceph-mon[65430]: from='osd.4 [v2:172.21.15.100:6800/3545662876,v1:172.21.15.100:6801/3545662876]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:11:32.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:32 smithi033 ceph-mon[65430]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:11:32.414 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:32 smithi033 ceph-mon[65430]: from='osd.4 ' entity='osd.4' 2024-09-17T01:11:32.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:32 smithi100 ceph-mon[51701]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 4.6 MiB/s wr, 473 op/s; 228/1521 objects degraded (14.990%) 2024-09-17T01:11:32.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:32 smithi100 ceph-mon[51701]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T01:11:32.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:32 smithi100 ceph-mon[51701]: osdmap e119: 8 total, 7 up, 8 in 2024-09-17T01:11:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:32 smithi100 ceph-mon[51701]: from='osd.4 [v2:172.21.15.100:6800/3545662876,v1:172.21.15.100:6801/3545662876]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:11:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:32 smithi100 ceph-mon[51701]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:11:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:32 smithi100 ceph-mon[51701]: from='osd.4 ' entity='osd.4' 2024-09-17T01:11:33.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195271 8.62 MB/sec execute 452 sec latency 633.028 ms 2024-09-17T01:11:33.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:33 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:11:33.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:33 smithi033 ceph-mon[65430]: osd.4 [v2:172.21.15.100:6800/3545662876,v1:172.21.15.100:6801/3545662876] boot 2024-09-17T01:11:33.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:33 smithi033 ceph-mon[65430]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T01:11:33.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:33 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:11:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:33 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:11:33.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:33 smithi100 ceph-mon[51701]: osd.4 [v2:172.21.15.100:6800/3545662876,v1:172.21.15.100:6801/3545662876] boot 2024-09-17T01:11:33.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:33 smithi100 ceph-mon[51701]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T01:11:33.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:33 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T01:11:34.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195271 8.60 MB/sec execute 453 sec latency 1633.058 ms 2024-09-17T01:11:34.983 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:34 smithi033 ceph-mon[65430]: pgmap v204: 97 pgs: 14 peering, 13 active+undersized, 14 active+undersized+degraded, 56 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.6 MiB/s wr, 579 op/s; 125/1551 objects degraded (8.059%) 2024-09-17T01:11:34.983 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:34 smithi033 ceph-mon[65430]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T01:11:34.984 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:34 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 125/1551 objects degraded (8.059%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:34 smithi100 ceph-mon[51701]: pgmap v204: 97 pgs: 14 peering, 13 active+undersized, 14 active+undersized+degraded, 56 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.6 MiB/s wr, 579 op/s; 125/1551 objects degraded (8.059%) 2024-09-17T01:11:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:34 smithi100 ceph-mon[51701]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T01:11:35.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:34 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 125/1551 objects degraded (8.059%), 14 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:35.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195368 8.59 MB/sec execute 454 sec latency 2250.700 ms 2024-09-17T01:11:35.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:35 smithi033 ceph-mon[65430]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T01:11:36.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:35 smithi100 ceph-mon[51701]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T01:11:36.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195514 8.59 MB/sec execute 455 sec latency 613.491 ms 2024-09-17T01:11:36.987 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:36 smithi033 ceph-mon[65430]: pgmap v207: 97 pgs: 16 peering, 11 active+undersized, 13 active+undersized+degraded, 57 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 421 op/s; 110/1581 objects degraded (6.958%) 2024-09-17T01:11:36.987 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:36 smithi033 ceph-mon[65430]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T01:11:37.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:36 smithi100 ceph-mon[51701]: pgmap v207: 97 pgs: 16 peering, 11 active+undersized, 13 active+undersized+degraded, 57 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 421 op/s; 110/1581 objects degraded (6.958%) 2024-09-17T01:11:37.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:36 smithi100 ceph-mon[51701]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T01:11:37.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195696 8.58 MB/sec execute 456 sec latency 1029.090 ms 2024-09-17T01:11:37.754 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:37 smithi033 ceph-mon[65430]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T01:11:37.754 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:37 smithi033 ceph-mon[65430]: pgmap v210: 97 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 16 peering, 2 active+recovery_wait+degraded, 75 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.6 MiB/s wr, 75 op/s; 82/1578 objects degraded (5.196%); 2.2 MiB/s, 65 keys/s, 20 objects/s recovering 2024-09-17T01:11:38.119 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:37 smithi100 ceph-mon[51701]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T01:11:38.119 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:37 smithi100 ceph-mon[51701]: pgmap v210: 97 pgs: 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 16 peering, 2 active+recovery_wait+degraded, 75 active+clean; 334 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.6 MiB/s wr, 75 op/s; 82/1578 objects degraded (5.196%); 2.2 MiB/s, 65 keys/s, 20 objects/s recovering 2024-09-17T01:11:38.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 195952 8.59 MB/sec execute 457 sec latency 317.386 ms 2024-09-17T01:11:38.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:38 smithi033 ceph-mon[65430]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T01:11:39.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:38 smithi100 ceph-mon[51701]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T01:11:39.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 196197 8.58 MB/sec execute 458 sec latency 766.053 ms 2024-09-17T01:11:39.800 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:39 smithi033 ceph-mon[65430]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T01:11:39.800 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:39 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 82/1578 objects degraded (5.196%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:39.800 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:39 smithi033 ceph-mon[65430]: pgmap v213: 97 pgs: 1 active+recovering+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 active+recovery_wait+degraded, 86 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.5 MiB/s wr, 190 op/s; 158/1620 objects degraded (9.753%); 2.7 MiB/s, 66 keys/s, 49 objects/s recovering 2024-09-17T01:11:40.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:39 smithi100 ceph-mon[51701]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T01:11:40.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:39 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 82/1578 objects degraded (5.196%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:40.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:39 smithi100 ceph-mon[51701]: pgmap v213: 97 pgs: 1 active+recovering+degraded, 1 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 2 active+recovery_wait+degraded, 86 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 511 B/s rd, 2.5 MiB/s wr, 190 op/s; 158/1620 objects degraded (9.753%); 2.7 MiB/s, 66 keys/s, 49 objects/s recovering 2024-09-17T01:11:40.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 196417 8.57 MB/sec execute 459 sec latency 799.481 ms 2024-09-17T01:11:40.842 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:40 smithi033 ceph-mon[65430]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T01:11:41.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:40 smithi100 ceph-mon[51701]: osdmap e127: 8 total, 8 up, 8 in 2024-09-17T01:11:41.345 INFO:teuthology.orchestra.run.smithi033.stdout: 5 196602 8.56 MB/sec execute 460 sec latency 639.658 ms 2024-09-17T01:11:41.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:41 smithi033 ceph-mon[65430]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T01:11:41.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:41 smithi033 ceph-mon[65430]: pgmap v216: 97 pgs: 1 active+recovering+degraded, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+degraded, 89 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.7 MiB/s wr, 180 op/s; 113/1668 objects degraded (6.775%); 552 KiB/s, 0 keys/s, 39 objects/s recovering 2024-09-17T01:11:41.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:41 smithi033 ceph-mon[65430]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T01:11:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:41 smithi100 ceph-mon[51701]: osdmap e128: 8 total, 8 up, 8 in 2024-09-17T01:11:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:41 smithi100 ceph-mon[51701]: pgmap v216: 97 pgs: 1 active+recovering+degraded, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+degraded, 89 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.7 MiB/s wr, 180 op/s; 113/1668 objects degraded (6.775%); 552 KiB/s, 0 keys/s, 39 objects/s recovering 2024-09-17T01:11:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:41 smithi100 ceph-mon[51701]: osdmap e129: 8 total, 8 up, 8 in 2024-09-17T01:11:42.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 196942 8.56 MB/sec execute 461 sec latency 64.858 ms 2024-09-17T01:11:43.139 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:43.139 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:43.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 197287 8.55 MB/sec execute 462 sec latency 69.629 ms 2024-09-17T01:11:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:44.163 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:44 smithi033 ceph-mon[65430]: pgmap v218: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.9 MiB/s wr, 446 op/s; 282 KiB/s, 38 objects/s recovering 2024-09-17T01:11:44.163 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:44 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 113/1668 objects degraded (6.775%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:44.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 197633 8.55 MB/sec execute 463 sec latency 50.505 ms 2024-09-17T01:11:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:44 smithi100 ceph-mon[51701]: pgmap v218: 97 pgs: 97 active+clean; 335 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.1 KiB/s rd, 3.9 MiB/s wr, 446 op/s; 282 KiB/s, 38 objects/s recovering 2024-09-17T01:11:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:44 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 113/1668 objects degraded (6.775%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:45.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:45 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 113/1668 objects degraded (6.775%), 6 pgs degraded) 2024-09-17T01:11:45.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:45 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:11:45.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:45 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 197918 8.54 MB/sec execute 464 sec latency 56.710 ms 2024-09-17T01:11:45.385 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:45 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 113/1668 objects degraded (6.775%), 6 pgs degraded) 2024-09-17T01:11:45.385 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:45 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:11:45.385 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:45 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:46.237 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:46 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:46.237 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:46 smithi033 ceph-mon[65430]: Upgrade: osd.5 is safe to restart 2024-09-17T01:11:46.237 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:46 smithi033 ceph-mon[65430]: pgmap v219: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.2 MiB/s wr, 341 op/s; 200 KiB/s, 27 objects/s recovering 2024-09-17T01:11:46.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 198358 8.56 MB/sec execute 465 sec latency 56.431 ms 2024-09-17T01:11:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:46 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T01:11:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:46 smithi100 ceph-mon[51701]: Upgrade: osd.5 is safe to restart 2024-09-17T01:11:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:46 smithi100 ceph-mon[51701]: pgmap v219: 97 pgs: 97 active+clean; 333 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 3.2 MiB/s wr, 341 op/s; 200 KiB/s, 27 objects/s recovering 2024-09-17T01:11:47.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 198731 8.56 MB/sec execute 466 sec latency 57.343 ms 2024-09-17T01:11:47.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:47 smithi033 ceph-mon[65430]: Upgrade: Updating osd.5 2024-09-17T01:11:47.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:47.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:11:47.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:11:47.749 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:47 smithi033 ceph-mon[65430]: Deploying daemon osd.5 on smithi100 2024-09-17T01:11:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:47 smithi100 ceph-mon[51701]: Upgrade: Updating osd.5 2024-09-17T01:11:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T01:11:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:11:47.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:47 smithi100 ceph-mon[51701]: Deploying daemon osd.5 on smithi100 2024-09-17T01:11:48.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 199046 8.57 MB/sec execute 467 sec latency 53.685 ms 2024-09-17T01:11:48.783 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:48 smithi033 ceph-mon[65430]: pgmap v220: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 330 op/s; 166 KiB/s, 22 objects/s recovering 2024-09-17T01:11:48.896 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:48 smithi100 ceph-mon[51701]: pgmap v220: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.2 MiB/s wr, 330 op/s; 166 KiB/s, 22 objects/s recovering 2024-09-17T01:11:49.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 199370 8.58 MB/sec execute 468 sec latency 43.247 ms 2024-09-17T01:11:50.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 199665 8.58 MB/sec execute 469 sec latency 95.025 ms 2024-09-17T01:11:50.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:50 smithi100 systemd[1]: Stopping Ceph osd.5 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:11:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:50 smithi033 ceph-mon[65430]: pgmap v221: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 560 op/s; 92 KiB/s, 14 objects/s recovering 2024-09-17T01:11:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:50 smithi033 ceph-mon[65430]: osd.5 marked itself down and dead 2024-09-17T01:11:50.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:50 smithi100 ceph-mon[51701]: pgmap v221: 97 pgs: 97 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.9 MiB/s wr, 560 op/s; 92 KiB/s, 14 objects/s recovering 2024-09-17T01:11:50.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:50 smithi100 ceph-mon[51701]: osd.5 marked itself down and dead 2024-09-17T01:11:50.978 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[33011]: 2024-09-17T01:11:50.544+0000 7fd973f67700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:11:50.978 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[33011]: 2024-09-17T01:11:50.544+0000 7fd973f67700 -1 osd.5 129 *** Got signal Terminated *** 2024-09-17T01:11:50.978 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[33011]: 2024-09-17T01:11:50.544+0000 7fd973f67700 -1 osd.5 129 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:11:51.353 INFO:teuthology.orchestra.run.smithi033.stdout: 5 199746 8.56 MB/sec execute 470 sec latency 641.878 ms 2024-09-17T01:11:51.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:51 smithi100 podman[58392]: 2024-09-17 01:11:51.348655208 +0000 UTC m=+0.930537397 container died 83cd9cb1d6b801f0adf455455a173b30cb44fc4e63c901abb692c466c3e4084c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, RELEASE=HEAD, ceph=True, name=centos-stream, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True) 2024-09-17T01:11:51.982 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:51 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:11:51.982 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:51 smithi033 ceph-mon[65430]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T01:11:51.982 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:51 smithi033 ceph-mon[65430]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T01:11:52.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:51 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:11:52.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:51 smithi100 ceph-mon[51701]: osdmap e130: 8 total, 7 up, 8 in 2024-09-17T01:11:52.126 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:51 smithi100 ceph-mon[51701]: osdmap e131: 8 total, 7 up, 8 in 2024-09-17T01:11:52.126 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:51 smithi100 podman[58392]: 2024-09-17 01:11:51.843918799 +0000 UTC m=+1.425800984 container cleanup 83cd9cb1d6b801f0adf455455a173b30cb44fc4e63c901abb692c466c3e4084c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, distribution-scope=public, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, GIT_BRANCH=HEAD, version=8) 2024-09-17T01:11:52.126 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:51 smithi100 bash[58392]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5 2024-09-17T01:11:52.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 199849 8.55 MB/sec execute 471 sec latency 1050.253 ms 2024-09-17T01:11:52.473 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58408]: 2024-09-17 01:11:52.12869826 +0000 UTC m=+0.780364970 container remove 83cd9cb1d6b801f0adf455455a173b30cb44fc4e63c901abb692c466c3e4084c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, version=8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, vcs-type=git, distribution-scope=public, io.buildah.version=1.19.8) 2024-09-17T01:11:52.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58491]: 2024-09-17 01:11:52.37895353 +0000 UTC m=+0.016051124 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:11:52.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58491]: 2024-09-17 01:11:52.513519168 +0000 UTC m=+0.150616747 container create 0a795199b61d1230b1b626c089676711c8df9a5609a768cea830a9c9975b053b (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T01:11:52.787 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:52 smithi033 ceph-mon[65430]: pgmap v223: 97 pgs: 13 stale+active+clean, 84 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 505 op/s; 0 B/s, 8 objects/s recovering 2024-09-17T01:11:52.787 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:53.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:52 smithi100 ceph-mon[51701]: pgmap v223: 97 pgs: 13 stale+active+clean, 84 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.5 MiB/s wr, 505 op/s; 0 B/s, 8 objects/s recovering 2024-09-17T01:11:53.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:53.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58491]: 2024-09-17 01:11:52.749637775 +0000 UTC m=+0.386735355 container init 0a795199b61d1230b1b626c089676711c8df9a5609a768cea830a9c9975b053b (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T01:11:53.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58491]: 2024-09-17 01:11:52.754234448 +0000 UTC m=+0.391332033 container start 0a795199b61d1230b1b626c089676711c8df9a5609a768cea830a9c9975b053b (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:11:53.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58491]: 2024-09-17 01:11:52.949827195 +0000 UTC m=+0.586924798 container attach 0a795199b61d1230b1b626c089676711c8df9a5609a768cea830a9c9975b053b (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-17T01:11:53.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:52 smithi100 podman[58491]: 2024-09-17 01:11:52.950187735 +0000 UTC m=+0.587285326 container died 0a795199b61d1230b1b626c089676711c8df9a5609a768cea830a9c9975b053b (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97) 2024-09-17T01:11:53.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 200078 8.54 MB/sec execute 472 sec latency 134.323 ms 2024-09-17T01:11:53.837 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:53 smithi033 ceph-mon[65430]: pgmap v225: 97 pgs: 3 active+undersized, 9 peering, 5 stale+active+clean, 8 active+undersized+degraded, 72 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.4 MiB/s wr, 387 op/s; 55/1632 objects degraded (3.370%) 2024-09-17T01:11:53.837 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:53 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 55/1632 objects degraded (3.370%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:54.169 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:53 smithi100 ceph-mon[51701]: pgmap v225: 97 pgs: 3 active+undersized, 9 peering, 5 stale+active+clean, 8 active+undersized+degraded, 72 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.4 MiB/s wr, 387 op/s; 55/1632 objects degraded (3.370%) 2024-09-17T01:11:54.170 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:53 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 55/1632 objects degraded (3.370%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:54.170 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:53 smithi100 podman[58515]: 2024-09-17 01:11:53.730375933 +0000 UTC m=+0.779398188 container remove 0a795199b61d1230b1b626c089676711c8df9a5609a768cea830a9c9975b053b (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:11:54.170 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:53 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T01:11:54.170 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:53 smithi100 systemd[1]: Stopped Ceph osd.5 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:11:54.170 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:53 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.5.service: Consumed 3min 49.289s CPU time. 2024-09-17T01:11:54.170 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:53 smithi100 systemd[1]: Starting Ceph osd.5 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:11:54.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 200274 8.53 MB/sec execute 473 sec latency 118.873 ms 2024-09-17T01:11:54.457 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 podman[58615]: 2024-09-17 01:11:54.074855467 +0000 UTC m=+0.017566351 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:11:54.457 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 podman[58615]: 2024-09-17 01:11:54.198907932 +0000 UTC m=+0.141618790 container create f7b35e2704656e00af1b00173700ca2665b38abb9f54e76a1c3447447ed1b17d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:11:54.457 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 podman[58615]: 2024-09-17 01:11:54.459857449 +0000 UTC m=+0.402568328 container init f7b35e2704656e00af1b00173700ca2665b38abb9f54e76a1c3447447ed1b17d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T01:11:54.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 podman[58615]: 2024-09-17 01:11:54.46423137 +0000 UTC m=+0.406942237 container start f7b35e2704656e00af1b00173700ca2665b38abb9f54e76a1c3447447ed1b17d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T01:11:54.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 podman[58615]: 2024-09-17 01:11:54.560039352 +0000 UTC m=+0.502750225 container attach f7b35e2704656e00af1b00173700ca2665b38abb9f54e76a1c3447447ed1b17d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, GIT_CLEAN=True) 2024-09-17T01:11:54.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:54.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 bash[58615]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:54.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:54.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:54 smithi100 bash[58615]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:11:55.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 200532 8.53 MB/sec execute 474 sec latency 78.161 ms 2024-09-17T01:11:55.977 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:55 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:11:55.977 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:55 smithi100 bash[58615]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:11:55.978 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:55 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:11:55.978 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:55 smithi100 bash[58615]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T01:11:56.188 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:56 smithi033 ceph-mon[65430]: pgmap v226: 97 pgs: 4 active+undersized, 9 peering, 15 active+undersized+degraded, 69 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 508 op/s; 115/1638 objects degraded (7.021%) 2024-09-17T01:11:56.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 200910 8.52 MB/sec execute 475 sec latency 54.196 ms 2024-09-17T01:11:56.403 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:56 smithi100 ceph-mon[51701]: pgmap v226: 97 pgs: 4 active+undersized, 9 peering, 15 active+undersized+degraded, 69 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.2 MiB/s wr, 508 op/s; 115/1638 objects degraded (7.021%) 2024-09-17T01:11:56.403 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:11:56.403 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 bash[58615]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T01:11:56.403 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 bash[58615]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 bash[58615]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 bash[58615]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate[58630]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 bash[58615]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T01:11:56.404 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 podman[58615]: 2024-09-17 01:11:56.041139533 +0000 UTC m=+1.983850400 container died f7b35e2704656e00af1b00173700ca2665b38abb9f54e76a1c3447447ed1b17d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T01:11:57.072 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:56 smithi100 podman[58772]: 2024-09-17 01:11:56.824132429 +0000 UTC m=+0.776718441 container remove f7b35e2704656e00af1b00173700ca2665b38abb9f54e76a1c3447447ed1b17d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-17T01:11:57.073 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 podman[58831]: 2024-09-17 01:11:56.948210212 +0000 UTC m=+0.018589568 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:11:57.073 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 podman[58831]: 2024-09-17 01:11:57.07539992 +0000 UTC m=+0.145779264 container create 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T01:11:57.350 INFO:teuthology.orchestra.run.smithi033.stdout: 5 201194 8.52 MB/sec execute 476 sec latency 85.690 ms 2024-09-17T01:11:57.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 podman[58831]: 2024-09-17 01:11:57.328250412 +0000 UTC m=+0.398629766 container init 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T01:11:57.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 podman[58831]: 2024-09-17 01:11:57.333543911 +0000 UTC m=+0.403923268 container start 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-17T01:11:57.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 ceph-osd[58850]: -- 172.21.15.100:0/2976308834 <== mon.1 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560e268e61e0 con 0x560e25adbc00 2024-09-17T01:11:57.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 bash[58831]: 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 2024-09-17T01:11:57.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:57 smithi100 systemd[1]: Started Ceph osd.5 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:11:58.231 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:58.231 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:58.231 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:58 smithi033 ceph-mon[65430]: pgmap v227: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.1 MiB/s wr, 250 op/s; 184/1608 objects degraded (11.443%) 2024-09-17T01:11:58.231 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:58.231 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:58.232 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:11:58.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 201441 8.51 MB/sec execute 477 sec latency 76.012 ms 2024-09-17T01:11:58.358 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:58.358 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:11:58.358 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:58 smithi100 ceph-mon[51701]: pgmap v227: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.1 MiB/s wr, 250 op/s; 184/1608 objects degraded (11.443%) 2024-09-17T01:11:58.358 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:58.358 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:11:58.358 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:11:59.250 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:11:59 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 184/1608 objects degraded (11.443%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:59.609 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:11:59 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 184/1608 objects degraded (11.443%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:11:59.609 INFO:teuthology.orchestra.run.smithi033.stdout: 5 201923 8.53 MB/sec execute 478 sec latency 62.330 ms 2024-09-17T01:11:59.609 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:11:59 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:11:59.216+0000 7f47d7130640 -1 Falling back to public interface 2024-09-17T01:11:59.795 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:12:00.267 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:00 smithi033 ceph-mon[65430]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.9 MiB/s wr, 409 op/s; 179/1617 objects degraded (11.070%) 2024-09-17T01:12:00.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 202360 8.54 MB/sec execute 479 sec latency 77.695 ms 2024-09-17T01:12:00.380 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:00 smithi100 ceph-mon[51701]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.9 MiB/s wr, 409 op/s; 179/1617 objects degraded (11.070%) 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (7m) 74s ago 16m 26.4M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (4m) 74s ago 16m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (4m) 35s ago 15m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (7m) 74s ago 16m 53.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (10m) 74s ago 10m 36.5M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (10m) 35s ago 10m 14.8M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (6m) 74s ago 17m 504M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (5m) 35s ago 15m 443M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:12:00.792 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (5m) 74s ago 17m 50.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (5m) 35s ago 15m 41.2M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (10m) 74s ago 10m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (10m) 35s ago 10m 88.5M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (7m) 74s ago 16m 15.0M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (6m) 35s ago 15m 16.6M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (3m) 74s ago 14m 331M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (3m) 74s ago 14m 269M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (2m) 74s ago 13m 188M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (85s) 74s ago 13m 14.5M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (39s) 35s ago 13m 14.2M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 starting - - - 2703M 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (12m) 35s ago 12m 353M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:12:00.793 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (11m) 35s ago 11m 558M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:12:00.794 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (6m) 74s ago 15m 62.7M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:12:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:01 smithi100 ceph-mon[51701]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:01.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:01 smithi100 ceph-mon[51701]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:01.281 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:01 smithi033 ceph-mon[65430]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:01.281 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:01 smithi033 ceph-mon[65430]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:01.344 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 9 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:12:01.345 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:12:01.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 202649 8.55 MB/sec execute 480 sec latency 53.855 ms 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:12:01.793 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:12:01.794 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:12:01.794 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:12:01.794 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T01:12:01.794 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:12:01.794 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:12:01.794 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:12:02.305 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:02 smithi033 ceph-mon[65430]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:02.305 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:02 smithi033 ceph-mon[65430]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 180/1599 objects degraded (11.257%) 2024-09-17T01:12:02.305 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:02 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3429850653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:02.305 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:02 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:02.305 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:02 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 180/1599 objects degraded (11.257%), 23 pgs degraded 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout: osd.5 (root=default,host=smithi100) is down 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 180/1599 objects degraded (11.257%), 23 pgs degraded 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-17T01:12:02.306 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-17T01:12:02.307 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-17T01:12:02.307 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-17T01:12:02.307 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-17T01:12:02.307 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-17T01:12:02.307 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-17T01:12:02.307 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-17T01:12:02.308 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-17T01:12:02.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 202886 8.55 MB/sec execute 481 sec latency 59.450 ms 2024-09-17T01:12:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:02 smithi100 ceph-mon[51701]: from='client.44291 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:02 smithi100 ceph-mon[51701]: pgmap v229: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 180/1599 objects degraded (11.257%) 2024-09-17T01:12:02.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:02 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3429850653' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:02 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:02.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:02 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:03 smithi100 ceph-mon[51701]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:03 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:03 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:03.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:03 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1793050321' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:12:03.316 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:03 smithi033 ceph-mon[65430]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:03.316 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:03 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:03.316 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:03 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:03.316 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:03 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1793050321' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:12:03.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 203215 8.54 MB/sec execute 482 sec latency 51.901 ms 2024-09-17T01:12:04.310 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:04 smithi033 ceph-mon[65430]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 375 op/s; 179/1611 objects degraded (11.111%) 2024-09-17T01:12:04.310 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:04 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 179/1611 objects degraded (11.111%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:04.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 203523 8.54 MB/sec execute 483 sec latency 42.522 ms 2024-09-17T01:12:04.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:04 smithi100 ceph-mon[51701]: pgmap v230: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 375 op/s; 179/1611 objects degraded (11.111%) 2024-09-17T01:12:04.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:04 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 179/1611 objects degraded (11.111%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:05.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 203747 8.53 MB/sec execute 484 sec latency 53.613 ms 2024-09-17T01:12:06.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:06 smithi100 ceph-mon[51701]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 433 op/s; 185/1641 objects degraded (11.274%) 2024-09-17T01:12:06.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 204021 8.53 MB/sec execute 485 sec latency 59.657 ms 2024-09-17T01:12:06.356 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:06 smithi033 ceph-mon[65430]: pgmap v231: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.2 MiB/s wr, 433 op/s; 185/1641 objects degraded (11.274%) 2024-09-17T01:12:07.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 204358 8.52 MB/sec execute 486 sec latency 61.493 ms 2024-09-17T01:12:07.391 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.391 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.391 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:12:07.391 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:07.392 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:07 smithi033 ceph-mon[65430]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T01:12:07.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:12:07.250+0000 7f47d7130640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:07.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:07.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:07 smithi100 ceph-mon[51701]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T01:12:08.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 204707 8.52 MB/sec execute 487 sec latency 55.547 ms 2024-09-17T01:12:08.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:08 smithi033 ceph-mon[65430]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 459 op/s; 187/1686 objects degraded (11.091%) 2024-09-17T01:12:08.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:08 smithi033 ceph-mon[65430]: from='osd.5 [v2:172.21.15.100:6808/953638171,v1:172.21.15.100:6809/953638171]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:12:08.409 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:08 smithi033 ceph-mon[65430]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:12:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:08 smithi100 ceph-mon[51701]: pgmap v232: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.5 MiB/s wr, 459 op/s; 187/1686 objects degraded (11.091%) 2024-09-17T01:12:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:08 smithi100 ceph-mon[51701]: from='osd.5 [v2:172.21.15.100:6808/953638171,v1:172.21.15.100:6809/953638171]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:12:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:08 smithi100 ceph-mon[51701]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T01:12:08.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:12:08 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:12:08.292+0000 7f47d7130640 -1 osd.5 129 log_to_monitors true 2024-09-17T01:12:08.728 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:12:08 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:12:08.382+0000 7f47ceedb640 -1 osd.5 129 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:12:09.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 204997 8.51 MB/sec execute 488 sec latency 39.479 ms 2024-09-17T01:12:09.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:09 smithi033 ceph-mon[65430]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:12:09.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:09 smithi033 ceph-mon[65430]: osdmap e132: 8 total, 7 up, 8 in 2024-09-17T01:12:09.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:09 smithi033 ceph-mon[65430]: from='osd.5 [v2:172.21.15.100:6808/953638171,v1:172.21.15.100:6809/953638171]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:09.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:09 smithi033 ceph-mon[65430]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:09.435 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:09 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 187/1686 objects degraded (11.091%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:09 smithi100 ceph-mon[51701]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T01:12:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:09 smithi100 ceph-mon[51701]: osdmap e132: 8 total, 7 up, 8 in 2024-09-17T01:12:09.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:09 smithi100 ceph-mon[51701]: from='osd.5 [v2:172.21.15.100:6808/953638171,v1:172.21.15.100:6809/953638171]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:09.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:09 smithi100 ceph-mon[51701]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:09.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:09 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 187/1686 objects degraded (11.091%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:10.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 205098 8.50 MB/sec execute 489 sec latency 922.385 ms 2024-09-17T01:12:10.639 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:10 smithi033 ceph-mon[65430]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 421 op/s; 182/1620 objects degraded (11.235%) 2024-09-17T01:12:10.639 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:10 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:12:10.639 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:10 smithi033 ceph-mon[65430]: osd.5 [v2:172.21.15.100:6808/953638171,v1:172.21.15.100:6809/953638171] boot 2024-09-17T01:12:10.639 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:10 smithi033 ceph-mon[65430]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T01:12:10.640 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:10 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:12:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:10 smithi100 ceph-mon[51701]: pgmap v234: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.4 MiB/s wr, 421 op/s; 182/1620 objects degraded (11.235%) 2024-09-17T01:12:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:10 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:12:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:10 smithi100 ceph-mon[51701]: osd.5 [v2:172.21.15.100:6808/953638171,v1:172.21.15.100:6809/953638171] boot 2024-09-17T01:12:10.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:10 smithi100 ceph-mon[51701]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T01:12:10.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:10 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T01:12:11.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 205322 8.50 MB/sec execute 490 sec latency 1922.416 ms 2024-09-17T01:12:11.452 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:11 smithi033 ceph-mon[65430]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T01:12:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:11 smithi100 ceph-mon[51701]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T01:12:12.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 205689 8.51 MB/sec execute 491 sec latency 2174.705 ms 2024-09-17T01:12:12.459 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:12 smithi033 ceph-mon[65430]: pgmap v237: 97 pgs: 3 peering, 7 active+undersized, 21 active+undersized+degraded, 66 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 152/1587 objects degraded (9.578%) 2024-09-17T01:12:12.459 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:12 smithi033 ceph-mon[65430]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T01:12:12.459 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:12.459 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:12 smithi100 ceph-mon[51701]: pgmap v237: 97 pgs: 3 peering, 7 active+undersized, 21 active+undersized+degraded, 66 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.7 MiB/s wr, 389 op/s; 152/1587 objects degraded (9.578%) 2024-09-17T01:12:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:12 smithi100 ceph-mon[51701]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T01:12:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:13.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 205912 8.51 MB/sec execute 492 sec latency 829.200 ms 2024-09-17T01:12:13.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:13 smithi033 ceph-mon[65430]: osdmap e136: 8 total, 8 up, 8 in 2024-09-17T01:12:13.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:13 smithi100 ceph-mon[51701]: osdmap e136: 8 total, 8 up, 8 in 2024-09-17T01:12:14.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 206131 8.51 MB/sec execute 493 sec latency 950.958 ms 2024-09-17T01:12:14.543 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:14 smithi033 ceph-mon[65430]: pgmap v240: 97 pgs: 1 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 3 active+undersized, 12 active+undersized+degraded, 77 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 MiB/s wr, 305 op/s; 112/1608 objects degraded (6.965%); 3.0 KiB/s, 8 objects/s recovering 2024-09-17T01:12:14.543 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:14 smithi033 ceph-mon[65430]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T01:12:14.543 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:14 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 112/1608 objects degraded (6.965%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:14 smithi100 ceph-mon[51701]: pgmap v240: 97 pgs: 1 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 3 active+undersized, 12 active+undersized+degraded, 77 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 MiB/s wr, 305 op/s; 112/1608 objects degraded (6.965%); 3.0 KiB/s, 8 objects/s recovering 2024-09-17T01:12:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:14 smithi100 ceph-mon[51701]: osdmap e137: 8 total, 8 up, 8 in 2024-09-17T01:12:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:14 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 112/1608 objects degraded (6.965%), 13 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:15.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 206325 8.50 MB/sec execute 494 sec latency 1079.264 ms 2024-09-17T01:12:15.556 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:15 smithi033 ceph-mon[65430]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T01:12:15.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:15 smithi100 ceph-mon[51701]: osdmap e138: 8 total, 8 up, 8 in 2024-09-17T01:12:16.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 206576 8.50 MB/sec execute 495 sec latency 51.741 ms 2024-09-17T01:12:16.574 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:16 smithi033 ceph-mon[65430]: pgmap v243: 97 pgs: 7 peering, 1 active+recovery_wait+degraded, 89 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 560 op/s; 16/1581 objects degraded (1.012%); 358 KiB/s, 38 objects/s recovering 2024-09-17T01:12:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:16 smithi100 ceph-mon[51701]: pgmap v243: 97 pgs: 7 peering, 1 active+recovery_wait+degraded, 89 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 560 op/s; 16/1581 objects degraded (1.012%); 358 KiB/s, 38 objects/s recovering 2024-09-17T01:12:17.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 206783 8.49 MB/sec execute 496 sec latency 47.259 ms 2024-09-17T01:12:17.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:17 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1581 objects degraded (1.012%), 1 pg degraded) 2024-09-17T01:12:17.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:17 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:12:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:17 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16/1581 objects degraded (1.012%), 1 pg degraded) 2024-09-17T01:12:17.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:17 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:12:18.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 206942 8.48 MB/sec execute 497 sec latency 99.301 ms 2024-09-17T01:12:18.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:18 smithi033 ceph-mon[65430]: pgmap v244: 97 pgs: 4 peering, 93 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.6 MiB/s wr, 293 op/s; 274 KiB/s, 37 objects/s recovering 2024-09-17T01:12:18.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:18 smithi100 ceph-mon[51701]: pgmap v244: 97 pgs: 4 peering, 93 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.6 MiB/s wr, 293 op/s; 274 KiB/s, 37 objects/s recovering 2024-09-17T01:12:19.352 INFO:teuthology.orchestra.run.smithi033.stdout: 5 207305 8.49 MB/sec execute 498 sec latency 37.390 ms 2024-09-17T01:12:20.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 207599 8.48 MB/sec execute 499 sec latency 64.305 ms 2024-09-17T01:12:20.661 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:20 smithi033 ceph-mon[65430]: pgmap v245: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 434 op/s; 1.8 MiB/s, 32 objects/s recovering 2024-09-17T01:12:20.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:20 smithi100 ceph-mon[51701]: pgmap v245: 97 pgs: 97 active+clean; 344 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.7 MiB/s wr, 434 op/s; 1.8 MiB/s, 32 objects/s recovering 2024-09-17T01:12:21.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 207915 8.47 MB/sec execute 500 sec latency 54.261 ms 2024-09-17T01:12:22.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 208196 8.47 MB/sec execute 501 sec latency 54.261 ms 2024-09-17T01:12:22.720 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:22 smithi033 ceph-mon[65430]: pgmap v246: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.6 MiB/s wr, 352 op/s; 1.5 MiB/s, 22 objects/s recovering 2024-09-17T01:12:22.720 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:22.720 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:22 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:22.720 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:22 smithi033 ceph-mon[65430]: Upgrade: osd.6 is safe to restart 2024-09-17T01:12:22.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:22 smithi100 ceph-mon[51701]: pgmap v246: 97 pgs: 97 active+clean; 340 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.6 MiB/s wr, 352 op/s; 1.5 MiB/s, 22 objects/s recovering 2024-09-17T01:12:22.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:22.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:22 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T01:12:22.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:22 smithi100 ceph-mon[51701]: Upgrade: osd.6 is safe to restart 2024-09-17T01:12:23.351 INFO:teuthology.orchestra.run.smithi033.stdout: 5 208605 8.48 MB/sec execute 502 sec latency 56.788 ms 2024-09-17T01:12:24.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 208892 8.47 MB/sec execute 503 sec latency 42.793 ms 2024-09-17T01:12:24.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:24 smithi033 ceph-mon[65430]: pgmap v247: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 1.2 MiB/s, 9 objects/s recovering 2024-09-17T01:12:24.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:24 smithi033 ceph-mon[65430]: Upgrade: Updating osd.6 2024-09-17T01:12:24.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:24.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:12:24.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:12:24.769 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:24 smithi033 ceph-mon[65430]: Deploying daemon osd.6 on smithi100 2024-09-17T01:12:24.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:24 smithi100 ceph-mon[51701]: pgmap v247: 97 pgs: 97 active+clean; 329 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 376 op/s; 1.2 MiB/s, 9 objects/s recovering 2024-09-17T01:12:24.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:24 smithi100 ceph-mon[51701]: Upgrade: Updating osd.6 2024-09-17T01:12:24.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:24.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T01:12:24.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:12:24.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:24 smithi100 ceph-mon[51701]: Deploying daemon osd.6 on smithi100 2024-09-17T01:12:25.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 209178 8.48 MB/sec execute 504 sec latency 64.149 ms 2024-09-17T01:12:26.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 209499 8.49 MB/sec execute 505 sec latency 73.136 ms 2024-09-17T01:12:26.813 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:26 smithi033 ceph-mon[65430]: pgmap v248: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 1002 KiB/s, 6 objects/s recovering 2024-09-17T01:12:26.835 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:26 smithi100 ceph-mon[51701]: pgmap v248: 97 pgs: 97 active+clean; 332 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.7 MiB/s wr, 362 op/s; 1002 KiB/s, 6 objects/s recovering 2024-09-17T01:12:27.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:27 smithi100 systemd[1]: Stopping Ceph osd.6 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:12:27.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 209847 8.49 MB/sec execute 506 sec latency 36.927 ms 2024-09-17T01:12:27.727 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:27 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[35977]: 2024-09-17T01:12:27.328+0000 7f69e7250700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:12:27.727 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:27 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[35977]: 2024-09-17T01:12:27.328+0000 7f69e7250700 -1 osd.6 138 *** Got signal Terminated *** 2024-09-17T01:12:27.728 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:27 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[35977]: 2024-09-17T01:12:27.328+0000 7f69e7250700 -1 osd.6 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:12:28.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:28.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:28.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:28 smithi033 ceph-mon[65430]: pgmap v249: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 358 op/s; 884 KiB/s, 5 objects/s recovering 2024-09-17T01:12:28.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:28.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:28 smithi033 ceph-mon[65430]: osd.6 marked itself down and dead 2024-09-17T01:12:28.344 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:28.344 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:28.344 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:28 smithi100 ceph-mon[51701]: pgmap v249: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 358 op/s; 884 KiB/s, 5 objects/s recovering 2024-09-17T01:12:28.344 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:28.344 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:28 smithi100 ceph-mon[51701]: osd.6 marked itself down and dead 2024-09-17T01:12:28.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 210153 8.49 MB/sec execute 507 sec latency 216.586 ms 2024-09-17T01:12:29.033 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:28 smithi100 podman[60642]: 2024-09-17 01:12:28.780964992 +0000 UTC m=+1.568613211 container died 905f83790ddf78757cf7013afe5b0b7631255e596673cf4b694a1c5b96f1ec62 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_BRANCH=HEAD, release=754, vendor=Red Hat, Inc., distribution-scope=public, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-17T01:12:29.286 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:29 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:12:29.286 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:29 smithi033 ceph-mon[65430]: osdmap e139: 8 total, 7 up, 8 in 2024-09-17T01:12:29.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 210210 8.47 MB/sec execute 508 sec latency 1043.197 ms 2024-09-17T01:12:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:29 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:12:29.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:29 smithi100 ceph-mon[51701]: osdmap e139: 8 total, 7 up, 8 in 2024-09-17T01:12:29.977 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:29 smithi100 podman[60642]: 2024-09-17 01:12:29.805850981 +0000 UTC m=+2.593499199 container cleanup 905f83790ddf78757cf7013afe5b0b7631255e596673cf4b694a1c5b96f1ec62 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, distribution-scope=public, name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, RELEASE=HEAD, release=754, maintainer=Guillaume Abrioux , version=8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-type=git, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T01:12:29.977 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:29 smithi100 bash[60642]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6 2024-09-17T01:12:30.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:30 smithi033 ceph-mon[65430]: pgmap v251: 97 pgs: 14 peering, 7 stale+active+clean, 76 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 405 op/s 2024-09-17T01:12:30.308 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:30 smithi033 ceph-mon[65430]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T01:12:30.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 210584 8.47 MB/sec execute 509 sec latency 91.328 ms 2024-09-17T01:12:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:30 smithi100 ceph-mon[51701]: pgmap v251: 97 pgs: 14 peering, 7 stale+active+clean, 76 active+clean; 336 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.3 MiB/s wr, 405 op/s 2024-09-17T01:12:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:30 smithi100 ceph-mon[51701]: osdmap e140: 8 total, 7 up, 8 in 2024-09-17T01:12:30.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:30 smithi100 podman[60669]: 2024-09-17 01:12:30.041148569 +0000 UTC m=+1.257715836 container remove 905f83790ddf78757cf7013afe5b0b7631255e596673cf4b694a1c5b96f1ec62 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-type=git, io.openshift.tags=base centos centos-stream, version=8, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, RELEASE=HEAD, release=754) 2024-09-17T01:12:30.794 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:30 smithi100 podman[60757]: 2024-09-17 01:12:30.398639971 +0000 UTC m=+0.018250734 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:12:30.794 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:30 smithi100 podman[60757]: 2024-09-17 01:12:30.525725026 +0000 UTC m=+0.145335823 container create f5f0074bb62beaaf003740514639c6ea8229fb4646d90afe0b4e81c6bb33f5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T01:12:30.794 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:30 smithi100 podman[60757]: 2024-09-17 01:12:30.796729558 +0000 UTC m=+0.416340324 container init f5f0074bb62beaaf003740514639c6ea8229fb4646d90afe0b4e81c6bb33f5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:12:31.203 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:30 smithi100 podman[60757]: 2024-09-17 01:12:30.801744299 +0000 UTC m=+0.421355054 container start f5f0074bb62beaaf003740514639c6ea8229fb4646d90afe0b4e81c6bb33f5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T01:12:31.203 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:30 smithi100 podman[60757]: 2024-09-17 01:12:30.9135978 +0000 UTC m=+0.533208559 container attach f5f0074bb62beaaf003740514639c6ea8229fb4646d90afe0b4e81c6bb33f5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:12:31.203 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:31 smithi100 podman[60757]: 2024-09-17 01:12:31.001048781 +0000 UTC m=+0.620659535 container died f5f0074bb62beaaf003740514639c6ea8229fb4646d90afe0b4e81c6bb33f5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2) 2024-09-17T01:12:31.326 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:31 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 12/1665 objects degraded (0.721%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:12:31.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 211022 8.48 MB/sec execute 510 sec latency 42.457 ms 2024-09-17T01:12:31.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:31 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 12/1665 objects degraded (0.721%), 1 pg degraded (PG_DEGRADED) 2024-09-17T01:12:31.851 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:31 smithi100 podman[60780]: 2024-09-17 01:12:31.626677246 +0000 UTC m=+0.619590882 container remove f5f0074bb62beaaf003740514639c6ea8229fb4646d90afe0b4e81c6bb33f5b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T01:12:31.851 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:31 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T01:12:31.851 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:31 smithi100 systemd[1]: Stopped Ceph osd.6 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:12:31.851 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:31 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.6.service: Consumed 3min 27.456s CPU time. 2024-09-17T01:12:32.213 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:31 smithi100 systemd[1]: Starting Ceph osd.6 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:12:32.213 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 podman[60877]: 2024-09-17 01:12:31.970173044 +0000 UTC m=+0.016569884 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:12:32.213 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 podman[60877]: 2024-09-17 01:12:32.096703206 +0000 UTC m=+0.143100039 container create 0756121dad3f3a3f94599dfafaf05a9578942e3eda058cb529b3cac76421d5f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T01:12:32.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:32 smithi033 ceph-mon[65430]: pgmap v253: 97 pgs: 14 peering, 6 stale+active+clean, 1 active+undersized+degraded, 76 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.3 MiB/s wr, 397 op/s; 12/1665 objects degraded (0.721%) 2024-09-17T01:12:32.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 211467 8.48 MB/sec execute 511 sec latency 51.842 ms 2024-09-17T01:12:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:32 smithi100 ceph-mon[51701]: pgmap v253: 97 pgs: 14 peering, 6 stale+active+clean, 1 active+undersized+degraded, 76 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.3 MiB/s wr, 397 op/s; 12/1665 objects degraded (0.721%) 2024-09-17T01:12:32.477 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 podman[60877]: 2024-09-17 01:12:32.33316937 +0000 UTC m=+0.379566228 container init 0756121dad3f3a3f94599dfafaf05a9578942e3eda058cb529b3cac76421d5f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:12:32.477 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 podman[60877]: 2024-09-17 01:12:32.34016394 +0000 UTC m=+0.386560779 container start 0756121dad3f3a3f94599dfafaf05a9578942e3eda058cb529b3cac76421d5f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T01:12:32.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 podman[60877]: 2024-09-17 01:12:32.441304942 +0000 UTC m=+0.487701787 container attach 0756121dad3f3a3f94599dfafaf05a9578942e3eda058cb529b3cac76421d5f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:12:32.830 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:12:32.977 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:12:32.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 bash[60877]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:12:32.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:12:32.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:32 smithi100 bash[60877]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:12:33.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 211795 8.48 MB/sec execute 512 sec latency 59.168 ms 2024-09-17T01:12:33.585 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:12:33.585 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:12:33.585 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:12:33.585 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (8m) 107s ago 16m 26.4M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (4m) 107s ago 16m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (4m) 32s ago 16m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (8m) 107s ago 16m 53.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (11m) 107s ago 11m 36.5M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (11m) 32s ago 11m 14.9M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (7m) 107s ago 18m 504M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (6m) 32s ago 16m 443M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (6m) 107s ago 18m 50.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:12:33.767 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (5m) 32s ago 16m 42.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (10m) 107s ago 10m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (10m) 32s ago 10m 88.6M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (8m) 107s ago 16m 15.0M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (7m) 32s ago 16m 16.8M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (4m) 107s ago 15m 331M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (3m) 107s ago 14m 269M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (2m) 107s ago 14m 188M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (118s) 107s ago 14m 14.5M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (72s) 32s ago 13m 252M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (36s) 32s ago 13m 11.1M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (12m) 32s ago 12m 384M 2703M 17.2.0 e1d6a67b021e 905f83790ddf 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (12m) 32s ago 12m 616M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:12:33.768 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (6m) 107s ago 15m 62.7M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:12:33.977 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate[60893]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 bash[60877]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T01:12:33.978 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:33 smithi100 podman[60877]: 2024-09-17 01:12:33.61901834 +0000 UTC m=+1.665415179 container died 0756121dad3f3a3f94599dfafaf05a9578942e3eda058cb529b3cac76421d5f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:12:34.302 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:34 smithi100 ceph-mon[51701]: from='client.44305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:34.302 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:34 smithi100 ceph-mon[51701]: pgmap v254: 97 pgs: 6 active+undersized, 14 peering, 15 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.6 MiB/s wr, 380 op/s; 108/1635 objects degraded (6.606%) 2024-09-17T01:12:34.302 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:34 smithi100 ceph-mon[51701]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:34.336 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:12:34.336 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:12:34.336 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:12:34.336 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T01:12:34.337 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 10 2024-09-17T01:12:34.338 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:12:34.338 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:12:34.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 212122 8.47 MB/sec execute 513 sec latency 66.804 ms 2024-09-17T01:12:34.394 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:34 smithi033 ceph-mon[65430]: from='client.44305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:34.394 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:34 smithi033 ceph-mon[65430]: pgmap v254: 97 pgs: 6 active+undersized, 14 peering, 15 active+undersized+degraded, 62 active+clean; 338 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 3.6 KiB/s rd, 4.6 MiB/s wr, 380 op/s; 108/1635 objects degraded (6.606%) 2024-09-17T01:12:34.395 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:34 smithi033 ceph-mon[65430]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:34.561 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 podman[61028]: 2024-09-17 01:12:34.304946068 +0000 UTC m=+0.679504745 container remove 0756121dad3f3a3f94599dfafaf05a9578942e3eda058cb529b3cac76421d5f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T01:12:34.561 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 podman[61082]: 2024-09-17 01:12:34.436697244 +0000 UTC m=+0.018784675 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:12:34.562 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 podman[61082]: 2024-09-17 01:12:34.564403832 +0000 UTC m=+0.146491241 container create 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-17T01:12:34.839 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:12:34.839 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:12:34.839 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:12:34.839 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:12:34.839 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:12:34.839 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:12:34.840 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:12:35.223 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 podman[61082]: 2024-09-17 01:12:34.842754186 +0000 UTC m=+0.424841608 container init 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T01:12:35.223 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 podman[61082]: 2024-09-17 01:12:34.846603999 +0000 UTC m=+0.428691411 container start 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:12:35.224 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 ceph-osd[61101]: -- 172.21.15.100:0/2075495825 <== mon.1 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5589653081e0 con 0x5589652de000 2024-09-17T01:12:35.224 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 bash[61082]: 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 2024-09-17T01:12:35.224 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:34 smithi100 systemd[1]: Started Ceph osd.6 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:12:35.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 212423 8.48 MB/sec execute 514 sec latency 50.595 ms 2024-09-17T01:12:35.412 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:35 smithi033 ceph-mon[65430]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:35.412 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:35 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1976861799' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:35.412 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:35 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:35.412 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:35 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:35.412 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:35 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 108/1635 objects degraded (6.606%), 15 pgs degraded 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout: osd.6 (root=default,host=smithi100) is down 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 108/1635 objects degraded (6.606%), 15 pgs degraded 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.7 is active+undersized+degraded, acting [7,2] 2024-09-17T01:12:35.446 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-17T01:12:35.447 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-17T01:12:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:35 smithi100 ceph-mon[51701]: from='client.34418 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:35 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1976861799' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:35 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:35.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:35 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:35.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:35 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:12:36.284 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:36 smithi033 ceph-mon[65430]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:36.284 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:36 smithi033 ceph-mon[65430]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 542 op/s; 191/1635 objects degraded (11.682%); 511 B/s, 0 objects/s recovering 2024-09-17T01:12:36.284 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:36 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2307105195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:12:36.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 212794 8.49 MB/sec execute 515 sec latency 152.974 ms 2024-09-17T01:12:36.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:36 smithi100 ceph-mon[51701]: from='client.34426 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:12:36.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:36 smithi100 ceph-mon[51701]: pgmap v255: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 333 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 4.9 MiB/s wr, 542 op/s; 191/1635 objects degraded (11.682%); 511 B/s, 0 objects/s recovering 2024-09-17T01:12:36.569 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:36 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2307105195' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:12:36.836 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:36 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:12:36.726+0000 7f7386975640 -1 Falling back to public interface 2024-09-17T01:12:37.313 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:37 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 188/1626 objects degraded (11.562%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:37.362 INFO:teuthology.orchestra.run.smithi033.stdout: 5 213164 8.50 MB/sec execute 516 sec latency 72.132 ms 2024-09-17T01:12:37.656 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:37 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 188/1626 objects degraded (11.562%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:38.361 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:38 smithi033 ceph-mon[65430]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.5 MiB/s wr, 425 op/s; 188/1626 objects degraded (11.562%); 460 B/s, 0 objects/s recovering 2024-09-17T01:12:38.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 213414 8.49 MB/sec execute 517 sec latency 144.265 ms 2024-09-17T01:12:38.552 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:38 smithi100 ceph-mon[51701]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.5 MiB/s wr, 425 op/s; 188/1626 objects degraded (11.562%); 460 B/s, 0 objects/s recovering 2024-09-17T01:12:39.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 213596 8.49 MB/sec execute 518 sec latency 81.045 ms 2024-09-17T01:12:40.188 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:40 smithi033 ceph-mon[65430]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 535 op/s; 199/1665 objects degraded (11.952%); 409 B/s, 0 objects/s recovering 2024-09-17T01:12:40.188 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:40 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.188 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:40 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.188 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:40 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.189 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:40 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 213757 8.48 MB/sec execute 519 sec latency 99.176 ms 2024-09-17T01:12:40.434 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:40 smithi100 ceph-mon[51701]: pgmap v257: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 342 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 535 op/s; 199/1665 objects degraded (11.952%); 409 B/s, 0 objects/s recovering 2024-09-17T01:12:40.434 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:40 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.434 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:40 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.434 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:40 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:40.434 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:40 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:41.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 214054 8.48 MB/sec execute 520 sec latency 70.603 ms 2024-09-17T01:12:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:42 smithi100 ceph-mon[51701]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.6 MiB/s wr, 461 op/s; 199/1647 objects degraded (12.083%); 344 B/s, 0 objects/s recovering 2024-09-17T01:12:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:42.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:42.251 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:42 smithi033 ceph-mon[65430]: pgmap v258: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 341 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.6 MiB/s wr, 461 op/s; 199/1647 objects degraded (12.083%); 344 B/s, 0 objects/s recovering 2024-09-17T01:12:42.251 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:42.251 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:42.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 214425 8.48 MB/sec execute 521 sec latency 32.039 ms 2024-09-17T01:12:43.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:43 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 202/1671 objects degraded (12.089%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:43.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 214793 8.48 MB/sec execute 522 sec latency 43.429 ms 2024-09-17T01:12:43.375 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:43 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 202/1671 objects degraded (12.089%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:44.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 215125 8.47 MB/sec execute 523 sec latency 73.487 ms 2024-09-17T01:12:44.557 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.3 MiB/s wr, 587 op/s; 202/1671 objects degraded (12.089%); 341 B/s, 0 objects/s recovering 2024-09-17T01:12:44.557 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.557 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:44.558 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:44 smithi033 ceph-mon[65430]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T01:12:44.727 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:12:44.260+0000 7f7386975640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T01:12:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: pgmap v259: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 5.3 MiB/s wr, 587 op/s; 202/1671 objects degraded (12.089%); 341 B/s, 0 objects/s recovering 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:44.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:44 smithi100 ceph-mon[51701]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T01:12:45.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 215397 8.47 MB/sec execute 524 sec latency 68.717 ms 2024-09-17T01:12:45.598 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:45 smithi033 ceph-mon[65430]: from='osd.6 [v2:172.21.15.100:6816/464651811,v1:172.21.15.100:6817/464651811]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:12:45.598 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:45 smithi033 ceph-mon[65430]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:12:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:45 smithi100 ceph-mon[51701]: from='osd.6 [v2:172.21.15.100:6816/464651811,v1:172.21.15.100:6817/464651811]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:12:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:45 smithi100 ceph-mon[51701]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T01:12:45.728 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:12:45.230+0000 7f7386975640 -1 osd.6 138 log_to_monitors true 2024-09-17T01:12:46.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:12:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:12:45.936+0000 7f737a718640 -1 osd.6 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:12:46.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 215671 8.46 MB/sec execute 525 sec latency 107.136 ms 2024-09-17T01:12:46.603 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:46 smithi033 ceph-mon[65430]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 509 op/s; 201/1659 objects degraded (12.116%); 341 B/s, 0 objects/s recovering 2024-09-17T01:12:46.603 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:46 smithi033 ceph-mon[65430]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:12:46.603 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:46 smithi033 ceph-mon[65430]: from='osd.6 [v2:172.21.15.100:6816/464651811,v1:172.21.15.100:6817/464651811]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:46.603 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:46 smithi033 ceph-mon[65430]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T01:12:46.603 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:46 smithi033 ceph-mon[65430]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:46.603 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:46 smithi033 ceph-mon[65430]: from='osd.6 ' entity='osd.6' 2024-09-17T01:12:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:46 smithi100 ceph-mon[51701]: pgmap v260: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 4.7 MiB/s wr, 509 op/s; 201/1659 objects degraded (12.116%); 341 B/s, 0 objects/s recovering 2024-09-17T01:12:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:46 smithi100 ceph-mon[51701]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T01:12:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:46 smithi100 ceph-mon[51701]: from='osd.6 [v2:172.21.15.100:6816/464651811,v1:172.21.15.100:6817/464651811]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:46 smithi100 ceph-mon[51701]: osdmap e141: 8 total, 7 up, 8 in 2024-09-17T01:12:46.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:46 smithi100 ceph-mon[51701]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:12:46.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:46 smithi100 ceph-mon[51701]: from='osd.6 ' entity='osd.6' 2024-09-17T01:12:47.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 216023 8.47 MB/sec execute 526 sec latency 415.772 ms 2024-09-17T01:12:47.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:47 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:12:47.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:47 smithi033 ceph-mon[65430]: osd.6 [v2:172.21.15.100:6816/464651811,v1:172.21.15.100:6817/464651811] boot 2024-09-17T01:12:47.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:47 smithi033 ceph-mon[65430]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T01:12:47.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:47 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:12:47.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:47 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:12:47.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:47 smithi100 ceph-mon[51701]: osd.6 [v2:172.21.15.100:6816/464651811,v1:172.21.15.100:6817/464651811] boot 2024-09-17T01:12:47.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:47 smithi100 ceph-mon[51701]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T01:12:47.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:47 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T01:12:48.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 216111 8.46 MB/sec execute 527 sec latency 1415.815 ms 2024-09-17T01:12:48.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:48 smithi033 ceph-mon[65430]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.1 MiB/s wr, 308 op/s; 193/1602 objects degraded (12.047%) 2024-09-17T01:12:48.526 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:48 smithi033 ceph-mon[65430]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T01:12:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:48 smithi100 ceph-mon[51701]: pgmap v263: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.1 MiB/s wr, 308 op/s; 193/1602 objects degraded (12.047%) 2024-09-17T01:12:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:48 smithi100 ceph-mon[51701]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T01:12:49.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 216272 8.45 MB/sec execute 528 sec latency 2049.637 ms 2024-09-17T01:12:49.670 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:49 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 193/1602 objects degraded (12.047%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:49.670 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:49 smithi033 ceph-mon[65430]: osdmap e144: 8 total, 8 up, 8 in 2024-09-17T01:12:49.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:49 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 193/1602 objects degraded (12.047%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:49.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:49 smithi100 ceph-mon[51701]: osdmap e144: 8 total, 8 up, 8 in 2024-09-17T01:12:50.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 216570 8.46 MB/sec execute 529 sec latency 405.310 ms 2024-09-17T01:12:50.477 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:50 smithi033 ceph-mon[65430]: pgmap v266: 97 pgs: 1 active+degraded, 6 remapped+peering, 7 active+undersized, 13 active+undersized+degraded, 70 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 4.0 MiB/s wr, 474 op/s; 102/1557 objects degraded (6.551%); 42 KiB/s, 10 objects/s recovering 2024-09-17T01:12:50.477 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:50 smithi033 ceph-mon[65430]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T01:12:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:50 smithi100 ceph-mon[51701]: pgmap v266: 97 pgs: 1 active+degraded, 6 remapped+peering, 7 active+undersized, 13 active+undersized+degraded, 70 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 4.0 MiB/s wr, 474 op/s; 102/1557 objects degraded (6.551%); 42 KiB/s, 10 objects/s recovering 2024-09-17T01:12:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:50 smithi100 ceph-mon[51701]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T01:12:51.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 216737 8.45 MB/sec execute 530 sec latency 1009.245 ms 2024-09-17T01:12:52.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:51 smithi033 ceph-mon[65430]: osdmap e146: 8 total, 8 up, 8 in 2024-09-17T01:12:52.052 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:51 smithi033 ceph-mon[65430]: pgmap v269: 97 pgs: 1 peering, 1 active+degraded, 6 remapped+peering, 6 active+undersized, 4 active+undersized+degraded, 79 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.8 MiB/s wr, 432 op/s; 56/1563 objects degraded (3.583%); 121 KiB/s, 18 objects/s recovering 2024-09-17T01:12:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:51 smithi100 ceph-mon[51701]: osdmap e146: 8 total, 8 up, 8 in 2024-09-17T01:12:52.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:51 smithi100 ceph-mon[51701]: pgmap v269: 97 pgs: 1 peering, 1 active+degraded, 6 remapped+peering, 6 active+undersized, 4 active+undersized+degraded, 79 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.8 MiB/s wr, 432 op/s; 56/1563 objects degraded (3.583%); 121 KiB/s, 18 objects/s recovering 2024-09-17T01:12:52.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 217032 8.45 MB/sec execute 531 sec latency 298.169 ms 2024-09-17T01:12:53.070 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:52 smithi033 ceph-mon[65430]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T01:12:53.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:52 smithi100 ceph-mon[51701]: osdmap e147: 8 total, 8 up, 8 in 2024-09-17T01:12:53.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 217285 8.44 MB/sec execute 532 sec latency 52.344 ms 2024-09-17T01:12:54.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:53 smithi033 ceph-mon[65430]: pgmap v271: 97 pgs: 1 peering, 4 remapped+peering, 92 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 371 op/s; 1.2 MiB/s, 55 keys/s, 34 objects/s recovering 2024-09-17T01:12:54.094 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:53 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 56/1563 objects degraded (3.583%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:53 smithi100 ceph-mon[51701]: pgmap v271: 97 pgs: 1 peering, 4 remapped+peering, 92 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.3 MiB/s wr, 371 op/s; 1.2 MiB/s, 55 keys/s, 34 objects/s recovering 2024-09-17T01:12:54.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:53 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 56/1563 objects degraded (3.583%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T01:12:54.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 217594 8.44 MB/sec execute 533 sec latency 58.788 ms 2024-09-17T01:12:55.117 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:54 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/1563 objects degraded (3.583%), 5 pgs degraded) 2024-09-17T01:12:55.117 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:54 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:12:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:54 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 56/1563 objects degraded (3.583%), 5 pgs degraded) 2024-09-17T01:12:55.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:54 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:12:55.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 217860 8.45 MB/sec execute 534 sec latency 48.043 ms 2024-09-17T01:12:56.142 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:55 smithi033 ceph-mon[65430]: pgmap v272: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 319 op/s; 2.4 MiB/s, 38 keys/s, 32 objects/s recovering 2024-09-17T01:12:56.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:55 smithi100 ceph-mon[51701]: pgmap v272: 97 pgs: 1 peering, 96 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 319 op/s; 2.4 MiB/s, 38 keys/s, 32 objects/s recovering 2024-09-17T01:12:56.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 218176 8.44 MB/sec execute 535 sec latency 56.953 ms 2024-09-17T01:12:57.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 218554 8.44 MB/sec execute 536 sec latency 75.370 ms 2024-09-17T01:12:58.209 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:58.209 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:58.209 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:58 smithi033 ceph-mon[65430]: pgmap v273: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.1 MiB/s wr, 312 op/s; 2.6 MiB/s, 32 keys/s, 27 objects/s recovering 2024-09-17T01:12:58.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 218882 8.44 MB/sec execute 537 sec latency 49.888 ms 2024-09-17T01:12:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:12:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:12:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:58 smithi100 ceph-mon[51701]: pgmap v273: 97 pgs: 97 active+clean; 337 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 4.1 MiB/s wr, 312 op/s; 2.6 MiB/s, 32 keys/s, 27 objects/s recovering 2024-09-17T01:12:59.225 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:59 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:59.225 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:59 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:59.225 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:12:59 smithi033 ceph-mon[65430]: Upgrade: osd.7 is safe to restart 2024-09-17T01:12:59.287 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:59 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:59.287 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:59 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T01:12:59.287 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:12:59 smithi100 ceph-mon[51701]: Upgrade: osd.7 is safe to restart 2024-09-17T01:12:59.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 219159 8.44 MB/sec execute 538 sec latency 62.508 ms 2024-09-17T01:13:00.283 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:00 smithi033 ceph-mon[65430]: pgmap v274: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.3 MiB/s wr, 413 op/s; 2.3 MiB/s, 28 keys/s, 19 objects/s recovering 2024-09-17T01:13:00.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 219503 8.44 MB/sec execute 539 sec latency 66.728 ms 2024-09-17T01:13:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:00 smithi100 ceph-mon[51701]: pgmap v274: 97 pgs: 97 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 4.3 MiB/s wr, 413 op/s; 2.3 MiB/s, 28 keys/s, 19 objects/s recovering 2024-09-17T01:13:01.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:01 smithi033 ceph-mon[65430]: Upgrade: Updating osd.7 2024-09-17T01:13:01.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:01.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:13:01.314 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:01 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:01.315 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:01 smithi033 ceph-mon[65430]: Deploying daemon osd.7 on smithi100 2024-09-17T01:13:01.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 219893 8.45 MB/sec execute 540 sec latency 63.108 ms 2024-09-17T01:13:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:01 smithi100 ceph-mon[51701]: Upgrade: Updating osd.7 2024-09-17T01:13:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T01:13:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:01 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:01 smithi100 ceph-mon[51701]: Deploying daemon osd.7 on smithi100 2024-09-17T01:13:02.296 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:02 smithi100 ceph-mon[51701]: pgmap v275: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 368 op/s; 1.8 MiB/s, 22 keys/s, 16 objects/s recovering 2024-09-17T01:13:02.296 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:02 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:02.324 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:02 smithi033 ceph-mon[65430]: pgmap v275: 97 pgs: 97 active+clean; 332 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 368 op/s; 1.8 MiB/s, 22 keys/s, 16 objects/s recovering 2024-09-17T01:13:02.324 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:02 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:02.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 220215 8.45 MB/sec execute 541 sec latency 70.218 ms 2024-09-17T01:13:03.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 220471 8.45 MB/sec execute 542 sec latency 54.476 ms 2024-09-17T01:13:03.727 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:03 smithi100 systemd[1]: Stopping Ceph osd.7 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:13:04.214 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:03 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[39033]: 2024-09-17T01:13:03.765+0000 7f86fe180700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:13:04.214 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:03 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[39033]: 2024-09-17T01:13:03.765+0000 7f86fe180700 -1 osd.7 147 *** Got signal Terminated *** 2024-09-17T01:13:04.214 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:03 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[39033]: 2024-09-17T01:13:03.765+0000 7f86fe180700 -1 osd.7 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:13:04.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:04 smithi033 ceph-mon[65430]: pgmap v276: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 465 op/s; 1.6 MiB/s, 8 objects/s recovering 2024-09-17T01:13:04.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:04 smithi033 ceph-mon[65430]: osd.7 marked itself down and dead 2024-09-17T01:13:04.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 220705 8.44 MB/sec execute 543 sec latency 145.030 ms 2024-09-17T01:13:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:04 smithi100 ceph-mon[51701]: pgmap v276: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.7 MiB/s wr, 465 op/s; 1.6 MiB/s, 8 objects/s recovering 2024-09-17T01:13:04.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:04 smithi100 ceph-mon[51701]: osd.7 marked itself down and dead 2024-09-17T01:13:05.147 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:04 smithi100 podman[62909]: 2024-09-17 01:13:04.886556044 +0000 UTC m=+1.248310688 container died 3af739620e482312399dc6cfe7886615e11afac7a1428aea7ce65f1130ee01ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, distribution-scope=public, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8, vcs-type=git, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, name=centos-stream, maintainer=Guillaume Abrioux , version=8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T01:13:05.363 INFO:teuthology.orchestra.run.smithi033.stdout: 5 220732 8.43 MB/sec execute 544 sec latency 1057.358 ms 2024-09-17T01:13:05.470 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:05 smithi033 ceph-mon[65430]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:13:05.471 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:05 smithi033 ceph-mon[65430]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T01:13:05.471 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:05 smithi033 ceph-mon[65430]: osdmap e148: 8 total, 7 up, 8 in 2024-09-17T01:13:05.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:05 smithi100 ceph-mon[51701]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T01:13:05.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:05 smithi100 ceph-mon[51701]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T01:13:05.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:05 smithi100 ceph-mon[51701]: osdmap e148: 8 total, 7 up, 8 in 2024-09-17T01:13:05.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:05 smithi100 podman[62909]: 2024-09-17 01:13:05.248885545 +0000 UTC m=+1.610640194 container cleanup 3af739620e482312399dc6cfe7886615e11afac7a1428aea7ce65f1130ee01ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, distribution-scope=public, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream) 2024-09-17T01:13:05.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:05 smithi100 bash[62909]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7 2024-09-17T01:13:05.934 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:05 smithi100 podman[62923]: 2024-09-17 01:13:05.575970006 +0000 UTC m=+0.688009620 container remove 3af739620e482312399dc6cfe7886615e11afac7a1428aea7ce65f1130ee01ed (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.buildah.version=1.19.8, release=754, distribution-scope=public, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, ceph=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, io.openshift.expose-services=, maintainer=Guillaume Abrioux , architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T01:13:06.046 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:13:06.227 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:05 smithi100 podman[63012]: 2024-09-17 01:13:05.839800409 +0000 UTC m=+0.017177151 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:13:06.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:05 smithi100 podman[63012]: 2024-09-17 01:13:05.985636437 +0000 UTC m=+0.163013154 container create 660d44a7268dd21903cbfb97e974ffc4b9a4d0e85807a6950adaff2e4e9f181c (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-17T01:13:06.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 221092 8.43 MB/sec execute 545 sec latency 49.573 ms 2024-09-17T01:13:06.483 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:06 smithi033 ceph-mon[65430]: pgmap v278: 97 pgs: 13 stale+active+clean, 84 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.1 MiB/s wr, 377 op/s; 410 KiB/s, 0 objects/s recovering 2024-09-17T01:13:06.483 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:06 smithi033 ceph-mon[65430]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T01:13:06.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:06 smithi100 ceph-mon[51701]: pgmap v278: 97 pgs: 13 stale+active+clean, 84 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.1 MiB/s wr, 377 op/s; 410 KiB/s, 0 objects/s recovering 2024-09-17T01:13:06.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:06 smithi100 ceph-mon[51701]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T01:13:06.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:06 smithi100 podman[63012]: 2024-09-17 01:13:06.246936733 +0000 UTC m=+0.424313470 container init 660d44a7268dd21903cbfb97e974ffc4b9a4d0e85807a6950adaff2e4e9f181c (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-17T01:13:06.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:06 smithi100 podman[63012]: 2024-09-17 01:13:06.250877369 +0000 UTC m=+0.428254089 container start 660d44a7268dd21903cbfb97e974ffc4b9a4d0e85807a6950adaff2e4e9f181c (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T01:13:06.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:06 smithi100 podman[63012]: 2024-09-17 01:13:06.346947451 +0000 UTC m=+0.524324180 container attach 660d44a7268dd21903cbfb97e974ffc4b9a4d0e85807a6950adaff2e4e9f181c (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:13:06.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:06 smithi100 podman[63012]: 2024-09-17 01:13:06.455435675 +0000 UTC m=+0.632812400 container died 660d44a7268dd21903cbfb97e974ffc4b9a4d0e85807a6950adaff2e4e9f181c (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T01:13:07.200 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (8m) 2m ago 17m 26.4M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (5m) 2m ago 17m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (5m) 28s ago 16m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (8m) 2m ago 17m 53.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (11m) 2m ago 11m 36.5M - 17.2.0 e1d6a67b021e 48ae5e26db6f 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (11m) 28s ago 11m 15.0M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (7m) 2m ago 18m 504M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (7m) 28s ago 16m 443M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (6m) 2m ago 18m 50.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (6m) 28s ago 16m 43.8M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (11m) 2m ago 11m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (11m) 28s ago 11m 88.6M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:13:07.201 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (8m) 2m ago 17m 15.0M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (8m) 28s ago 16m 16.5M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (4m) 2m ago 16m 331M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (4m) 2m ago 15m 269M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (3m) 2m ago 15m 188M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (2m) 2m ago 14m 14.5M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (106s) 28s ago 14m 278M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (70s) 28s ago 13m 245M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (32s) 28s ago 13m 11.0M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 82283e99afeb 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (13m) 28s ago 13m 659M 2703M 17.2.0 e1d6a67b021e 3af739620e48 2024-09-17T01:13:07.202 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (7m) 2m ago 16m 62.7M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:13:07.261 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:07 smithi033 ceph-mon[65430]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:07.261 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:07 smithi033 ceph-mon[65430]: from='client.44329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:07.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 221398 8.44 MB/sec execute 546 sec latency 64.515 ms 2024-09-17T01:13:07.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:07 smithi100 ceph-mon[51701]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:07.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:07 smithi100 ceph-mon[51701]: from='client.44329 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:07.833 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:13:07.833 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:13:07.833 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:13:07.833 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:07.833 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:13:07.834 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:13:07.834 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:07.834 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:13:07.834 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T01:13:07.834 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:07.834 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:13:07.835 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T01:13:07.835 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:07.835 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:13:07.835 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T01:13:07.835 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 11 2024-09-17T01:13:07.835 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:13:07.836 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:13:07.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:07 smithi100 podman[63035]: 2024-09-17 01:13:07.535859821 +0000 UTC m=+1.070314954 container remove 660d44a7268dd21903cbfb97e974ffc4b9a4d0e85807a6950adaff2e4e9f181c (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T01:13:07.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:07 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T01:13:07.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:07 smithi100 systemd[1]: Stopped Ceph osd.7 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:13:07.979 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:07 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.7.service: Consumed 5min 19.785s CPU time. 2024-09-17T01:13:07.979 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:07 smithi100 systemd[1]: Starting Ceph osd.7 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:13:08.244 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:07 smithi100 podman[63129]: 2024-09-17 01:13:07.89065121 +0000 UTC m=+0.019548342 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:13:08.244 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 podman[63129]: 2024-09-17 01:13:08.016293388 +0000 UTC m=+0.145190485 container create 3acca260a306994965c56bf37e036a510c2f218ad80bfe7f1d74e842e4eda875 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-c2ddf97) 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "13/23 daemons upgraded", 2024-09-17T01:13:08.268 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T01:13:08.269 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:13:08.269 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:13:08.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:08 smithi033 ceph-mon[65430]: pgmap v280: 97 pgs: 9 peering, 3 stale+active+clean, 16 active+undersized+degraded, 69 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 488 op/s; 115/1590 objects degraded (7.233%) 2024-09-17T01:13:08.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:08 smithi033 ceph-mon[65430]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:08.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:08 smithi033 ceph-mon[65430]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-17T01:13:08.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:08 smithi033 ceph-mon[65430]: Health check failed: Degraded data redundancy: 115/1590 objects degraded (7.233%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:08.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:08 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1965049621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:08.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 221723 8.44 MB/sec execute 547 sec latency 55.960 ms 2024-09-17T01:13:08.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-mon[51701]: pgmap v280: 97 pgs: 9 peering, 3 stale+active+clean, 16 active+undersized+degraded, 69 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.5 MiB/s wr, 488 op/s; 115/1590 objects degraded (7.233%) 2024-09-17T01:13:08.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-mon[51701]: from='client.44333 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:08.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-mon[51701]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-17T01:13:08.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-mon[51701]: Health check failed: Degraded data redundancy: 115/1590 objects degraded (7.233%), 16 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:08.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1965049621' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:08.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 podman[63129]: 2024-09-17 01:13:08.273941357 +0000 UTC m=+0.402838458 container init 3acca260a306994965c56bf37e036a510c2f218ad80bfe7f1d74e842e4eda875 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T01:13:08.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 podman[63129]: 2024-09-17 01:13:08.278376267 +0000 UTC m=+0.407273377 container start 3acca260a306994965c56bf37e036a510c2f218ad80bfe7f1d74e842e4eda875 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T01:13:08.729 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 podman[63129]: 2024-09-17 01:13:08.398888379 +0000 UTC m=+0.527785496 container attach 3acca260a306994965c56bf37e036a510c2f218ad80bfe7f1d74e842e4eda875 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:13:08.729 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:13:08.729 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 bash[63129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:13:08.729 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:13:08.729 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 bash[63129]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T01:13:08.812 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Reduced data availability: 2 pgs peering; Degraded data redundancy: 115/1590 objects degraded (7.233%), 16 pgs degraded 2024-09-17T01:13:08.812 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T01:13:08.812 INFO:teuthology.orchestra.run.smithi033.stdout: osd.7 (root=default,host=smithi100) is down 2024-09-17T01:13:08.812 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs peering 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 1.0 is stuck peering for 6m, current state peering, last acting [0,6] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.8 is stuck peering for 2m, current state peering, last acting [0,1] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 115/1590 objects degraded (7.233%), 16 pgs degraded 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.b is active+undersized+degraded, acting [4,5] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1a is active+undersized+degraded, acting [6,4] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 2.1d is active+undersized+degraded, acting [6,0] 2024-09-17T01:13:08.813 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.9 is active+undersized+degraded, acting [4,2] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.e is active+undersized+degraded, acting [4,1] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.f is active+undersized+degraded, acting [4,0] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.11 is active+undersized+degraded, acting [4,6] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.13 is active+undersized+degraded, acting [4,2] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 3.14 is active+undersized+degraded, acting [4,6] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.5 is active+undersized+degraded, acting [6,4] 2024-09-17T01:13:08.814 INFO:teuthology.orchestra.run.smithi033.stdout: pg 4.a is active+undersized+degraded, acting [6,1] 2024-09-17T01:13:09.215 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:13:09.215 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 bash[63129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:13:09.215 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:13:09.215 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:08 smithi100 bash[63129]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T01:13:09.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:09 smithi033 ceph-mon[65430]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:09.350 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:09 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/774963344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:13:09.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 222038 8.43 MB/sec execute 548 sec latency 58.862 ms 2024-09-17T01:13:09.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-mon[51701]: from='client.34450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:09.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/774963344' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 bash[63129]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 bash[63129]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 bash[63129]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T01:13:09.478 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:13:09.479 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 bash[63129]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T01:13:09.479 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate[63144]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T01:13:09.479 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 bash[63129]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T01:13:09.479 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:09 smithi100 podman[63129]: 2024-09-17 01:13:09.256720646 +0000 UTC m=+1.385617756 container died 3acca260a306994965c56bf37e036a510c2f218ad80bfe7f1d74e842e4eda875 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T01:13:10.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 222378 8.42 MB/sec execute 549 sec latency 46.944 ms 2024-09-17T01:13:10.371 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:10 smithi033 ceph-mon[65430]: pgmap v281: 97 pgs: 4 active+undersized, 9 peering, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s wr, 506 op/s; 203/1596 objects degraded (12.719%) 2024-09-17T01:13:10.581 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:10 smithi100 ceph-mon[51701]: pgmap v281: 97 pgs: 4 active+undersized, 9 peering, 28 active+undersized+degraded, 56 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s wr, 506 op/s; 203/1596 objects degraded (12.719%) 2024-09-17T01:13:10.581 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:10 smithi100 podman[63284]: 2024-09-17 01:13:10.346090741 +0000 UTC m=+1.082104610 container remove 3acca260a306994965c56bf37e036a510c2f218ad80bfe7f1d74e842e4eda875 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T01:13:10.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:10 smithi100 podman[63346]: 2024-09-17 01:13:10.487028656 +0000 UTC m=+0.018101338 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:13:10.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:10 smithi100 podman[63346]: 2024-09-17 01:13:10.605511066 +0000 UTC m=+0.136583731 container create edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-17T01:13:11.301 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:11 smithi100 podman[63346]: 2024-09-17 01:13:11.141833918 +0000 UTC m=+0.672906586 container init edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-17T01:13:11.301 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:11 smithi100 podman[63346]: 2024-09-17 01:13:11.1455035 +0000 UTC m=+0.676576172 container start edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T01:13:11.301 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:11 smithi100 ceph-osd[63365]: -- 172.21.15.100:0/2474460733 <== mon.1 v2:172.21.15.100:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bf7855a1e0 con 0x55bf78538000 2024-09-17T01:13:11.301 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:11 smithi100 bash[63346]: edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d 2024-09-17T01:13:11.302 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:11 smithi100 systemd[1]: Started Ceph osd.7 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:13:11.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 222636 8.42 MB/sec execute 550 sec latency 44.900 ms 2024-09-17T01:13:11.387 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:11 smithi033 ceph-mon[65430]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-17T01:13:11.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:11 smithi100 ceph-mon[51701]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-17T01:13:12.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 223027 8.43 MB/sec execute 551 sec latency 36.819 ms 2024-09-17T01:13:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:12 smithi033 ceph-mon[65430]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.5 MiB/s wr, 332 op/s; 252/1620 objects degraded (15.556%) 2024-09-17T01:13:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:12.424 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:12.425 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:12 smithi100 ceph-mon[51701]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.5 MiB/s wr, 332 op/s; 252/1620 objects degraded (15.556%) 2024-09-17T01:13:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:12.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:13.149 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:13 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:13:13.016+0000 7f02e232e640 -1 Falling back to public interface 2024-09-17T01:13:13.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 223440 8.44 MB/sec execute 552 sec latency 71.585 ms 2024-09-17T01:13:13.442 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:13 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 254/1620 objects degraded (15.679%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:13.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:13 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 254/1620 objects degraded (15.679%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:14.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 223761 8.44 MB/sec execute 553 sec latency 51.088 ms 2024-09-17T01:13:14.467 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:14 smithi033 ceph-mon[65430]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s wr, 363 op/s; 254/1620 objects degraded (15.679%) 2024-09-17T01:13:14.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:14 smithi100 ceph-mon[51701]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s wr, 363 op/s; 254/1620 objects degraded (15.679%) 2024-09-17T01:13:15.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 224073 8.44 MB/sec execute 554 sec latency 44.271 ms 2024-09-17T01:13:16.181 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:16 smithi033 ceph-mon[65430]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 522 op/s; 255/1617 objects degraded (15.770%) 2024-09-17T01:13:16.181 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.181 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.181 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.181 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 224304 8.43 MB/sec execute 555 sec latency 36.713 ms 2024-09-17T01:13:16.405 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:16 smithi100 ceph-mon[51701]: pgmap v284: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.5 MiB/s wr, 522 op/s; 255/1617 objects degraded (15.770%) 2024-09-17T01:13:16.405 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.405 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.405 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:16.405 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:17.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 224565 8.43 MB/sec execute 556 sec latency 68.472 ms 2024-09-17T01:13:18.214 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:18 smithi033 ceph-mon[65430]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 478 op/s; 262/1656 objects degraded (15.821%) 2024-09-17T01:13:18.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 224816 8.43 MB/sec execute 557 sec latency 82.866 ms 2024-09-17T01:13:18.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:18 smithi100 ceph-mon[51701]: pgmap v285: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 341 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 478 op/s; 262/1656 objects degraded (15.821%) 2024-09-17T01:13:19.242 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:19 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 262/1656 objects degraded (15.821%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:19.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 225108 8.43 MB/sec execute 558 sec latency 56.329 ms 2024-09-17T01:13:19.443 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:19 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 262/1656 objects degraded (15.821%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 391 op/s; 261/1668 objects degraded (15.647%) 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.256 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T01:13:20.257 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T01:13:20.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T01:13:20.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T01:13:20.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T01:13:20.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T01:13:20.258 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:20 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T01:13:20.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 225496 8.43 MB/sec execute 559 sec latency 60.863 ms 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: pgmap v286: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 344 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.5 MiB/s wr, 391 op/s; 261/1668 objects degraded (15.647%) 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:20.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T01:13:20.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T01:13:20.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T01:13:20.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T01:13:20.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:20 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T01:13:21.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:21 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all osd 2024-09-17T01:13:21.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:21 smithi033 ceph-mon[65430]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T01:13:21.290 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:21 smithi033 ceph-mon[65430]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T01:13:21.290 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:21 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T01:13:21.290 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:21 smithi033 ceph-mon[65430]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T01:13:21.290 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:21 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi033.gofagb"]}]: dispatch 2024-09-17T01:13:21.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 225785 8.42 MB/sec execute 560 sec latency 83.432 ms 2024-09-17T01:13:21.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all osd 2024-09-17T01:13:21.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-mon[51701]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T01:13:21.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-mon[51701]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T01:13:21.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T01:13:21.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-mon[51701]: osdmap e150: 8 total, 7 up, 8 in 2024-09-17T01:13:21.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi033.gofagb"]}]: dispatch 2024-09-17T01:13:21.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:21 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:13:21.564+0000 7f02e232e640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T01:13:22.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:22 smithi033 ceph-mon[65430]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.4 MiB/s wr, 444 op/s; 259/1641 objects degraded (15.783%) 2024-09-17T01:13:22.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:22 smithi033 ceph-mon[65430]: Upgrade: It appears safe to stop mds.foofs.smithi033.gofagb 2024-09-17T01:13:22.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226052 8.42 MB/sec execute 561 sec latency 78.528 ms 2024-09-17T01:13:22.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:22 smithi100 ceph-mon[51701]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.4 MiB/s wr, 444 op/s; 259/1641 objects degraded (15.783%) 2024-09-17T01:13:22.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:22 smithi100 ceph-mon[51701]: Upgrade: It appears safe to stop mds.foofs.smithi033.gofagb 2024-09-17T01:13:23.168 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:22 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:13:22.824+0000 7f02e232e640 -1 osd.7 147 log_to_monitors true 2024-09-17T01:13:23.347 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:23 smithi033 ceph-mon[65430]: from='osd.7 [v2:172.21.15.100:6824/4045410715,v1:172.21.15.100:6825/4045410715]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:13:23.347 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:23 smithi033 ceph-mon[65430]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:13:23.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226369 8.42 MB/sec execute 562 sec latency 68.049 ms 2024-09-17T01:13:23.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:23 smithi100 ceph-mon[51701]: from='osd.7 [v2:172.21.15.100:6824/4045410715,v1:172.21.15.100:6825/4045410715]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:13:23.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:23 smithi100 ceph-mon[51701]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T01:13:23.480 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:13:23 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:13:23.222+0000 7f02da0d9640 -1 osd.7 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T01:13:24.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 542 op/s; 245/1596 objects degraded (15.351%) 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: from='osd.7 [v2:172.21.15.100:6824/4045410715,v1:172.21.15.100:6825/4045410715]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: Upgrade: Updating mds.foofs.smithi033.gofagb 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: Deploying daemon mds.foofs.smithi033.gofagb on smithi033 2024-09-17T01:13:24.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:24 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 245/1596 objects degraded (15.351%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:24.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.43 MB/sec execute 563 sec latency 171.845 ms 2024-09-17T01:13:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: pgmap v289: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 343 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.0 MiB/s wr, 542 op/s; 245/1596 objects degraded (15.351%) 2024-09-17T01:13:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T01:13:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: from='osd.7 [v2:172.21.15.100:6824/4045410715,v1:172.21.15.100:6825/4045410715]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: osdmap e151: 8 total, 7 up, 8 in 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi100", "root=default"]}]: dispatch 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: Upgrade: Updating mds.foofs.smithi033.gofagb 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi033.gofagb", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: Deploying daemon mds.foofs.smithi033.gofagb on smithi033 2024-09-17T01:13:24.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:24 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 245/1596 objects degraded (15.351%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:25.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.42 MB/sec execute 564 sec latency 1171.893 ms 2024-09-17T01:13:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:25 smithi100 ceph-mon[51701]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:13:25.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:25 smithi100 ceph-mon[51701]: osd.7 [v2:172.21.15.100:6824/4045410715,v1:172.21.15.100:6825/4045410715] boot 2024-09-17T01:13:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:25 smithi100 ceph-mon[51701]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T01:13:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:25 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:13:25.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:25 smithi100 ceph-mon[51701]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T01:13:25.488 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:25 smithi033 ceph-mon[65430]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T01:13:25.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:25 smithi033 ceph-mon[65430]: osd.7 [v2:172.21.15.100:6824/4045410715,v1:172.21.15.100:6825/4045410715] boot 2024-09-17T01:13:25.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:25 smithi033 ceph-mon[65430]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T01:13:25.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:25 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T01:13:25.489 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:25 smithi033 ceph-mon[65430]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T01:13:26.363 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:13:26.074+0000 7f4c155c5640 -1 bdev(0x559045368c00 /var/lib/ceph/osd/ceph-2/block) aio_submit retries 1 2024-09-17T01:13:26.363 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:13:26.078+0000 7f4c12dc0640 -1 bdev(0x559045368c00 /var/lib/ceph/osd/ceph-2/block) aio_submit retries 11 2024-09-17T01:13:26.364 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:13:26.078+0000 7f4c13dc2640 -1 bdev(0x559045368c00 /var/lib/ceph/osd/ceph-2/block) aio_submit retries 10 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:25 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[65426]: 2024-09-17T01:13:25.979+0000 7fe2bf6d3640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.4 MiB/s wr, 427 op/s; 248/1596 objects degraded (15.539%) 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: Standby daemon mds.foofs.smithi100.ssfebu assigned to filesystem foofs as rank 0 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T01:13:26.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:26 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:replay} 2024-09-17T01:13:26.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.40 MB/sec execute 565 sec latency 2171.936 ms 2024-09-17T01:13:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: pgmap v292: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 3.4 MiB/s wr, 427 op/s; 248/1596 objects degraded (15.539%) 2024-09-17T01:13:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T01:13:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:13:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T01:13:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: Standby daemon mds.foofs.smithi100.ssfebu assigned to filesystem foofs as rank 0 2024-09-17T01:13:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T01:13:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T01:13:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T01:13:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:26 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:replay} 2024-09-17T01:13:27.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.39 MB/sec execute 566 sec latency 3171.983 ms 2024-09-17T01:13:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:28 smithi033 ceph-mon[65430]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T01:13:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:28.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:28 smithi033 ceph-mon[65430]: pgmap v296: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 unknown, 5 peering, 4 active+undersized, 21 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.6 MiB/s wr, 409 op/s; 194/1623 objects degraded (11.953%); 1.6 MiB/s, 13 objects/s recovering 2024-09-17T01:13:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:28 smithi100 ceph-mon[51701]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T01:13:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:28 smithi100 ceph-mon[51701]: pgmap v296: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 unknown, 5 peering, 4 active+undersized, 21 active+undersized+degraded, 62 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 4.6 MiB/s wr, 409 op/s; 194/1623 objects degraded (11.953%); 1.6 MiB/s, 13 objects/s recovering 2024-09-17T01:13:28.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.37 MB/sec execute 567 sec latency 4172.029 ms 2024-09-17T01:13:29.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T01:13:29.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T01:13:29.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: reconnect by client.24361 172.21.15.100:0/3113584054 after 0.000999974 2024-09-17T01:13:29.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:reconnect 2024-09-17T01:13:29.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.33:6834/4014687254,v1:172.21.15.33:6835/4014687254] up:boot 2024-09-17T01:13:29.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:reconnect} 1 up:standby 2024-09-17T01:13:29.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:13:29.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: reconnect by client.14540 172.21.15.33:0/2965077002 after 0.0649983 2024-09-17T01:13:29.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:29 smithi033 ceph-mon[65430]: Health check update: Degraded data redundancy: 194/1623 objects degraded (11.953%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:29.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.36 MB/sec execute 568 sec latency 5172.071 ms 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: reconnect by client.24361 172.21.15.100:0/3113584054 after 0.000999974 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:reconnect 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.33:6834/4014687254,v1:172.21.15.33:6835/4014687254] up:boot 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:reconnect} 1 up:standby 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: reconnect by client.14540 172.21.15.33:0/2965077002 after 0.0649983 2024-09-17T01:13:29.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:29 smithi100 ceph-mon[51701]: Health check update: Degraded data redundancy: 194/1623 objects degraded (11.953%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T01:13:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:30 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:rejoin 2024-09-17T01:13:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:30 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:rejoin} 1 up:standby 2024-09-17T01:13:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:30 smithi033 ceph-mon[65430]: pgmap v298: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 unknown, 5 peering, 1 active+undersized+degraded, 88 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 3.3 MiB/s wr, 328 op/s; 27/1632 objects degraded (1.654%); 4.3 MiB/s, 57 objects/s recovering 2024-09-17T01:13:30.364 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226707 8.34 MB/sec execute 569 sec latency 6172.114 ms 2024-09-17T01:13:30.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:30 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:rejoin 2024-09-17T01:13:30.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:30 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:rejoin} 1 up:standby 2024-09-17T01:13:30.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:30 smithi100 ceph-mon[51701]: pgmap v298: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 unknown, 5 peering, 1 active+undersized+degraded, 88 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 28 MiB/s rd, 3.3 MiB/s wr, 328 op/s; 27/1632 objects degraded (1.654%); 4.3 MiB/s, 57 objects/s recovering 2024-09-17T01:13:31.301 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:31 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:clientreplay 2024-09-17T01:13:31.301 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:31 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:clientreplay} 1 up:standby 2024-09-17T01:13:31.301 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:31 smithi033 ceph-mon[65430]: daemon mds.foofs.smithi100.ssfebu is now active in filesystem foofs as rank 0 2024-09-17T01:13:31.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226762 8.33 MB/sec execute 570 sec latency 6878.756 ms 2024-09-17T01:13:31.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:31 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:clientreplay 2024-09-17T01:13:31.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:31 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi100.ssfebu=up:clientreplay} 1 up:standby 2024-09-17T01:13:31.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:31 smithi100 ceph-mon[51701]: daemon mds.foofs.smithi100.ssfebu is now active in filesystem foofs as rank 0 2024-09-17T01:13:32.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:32 smithi033 ceph-mon[65430]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T01:13:32.099 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:32 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:active 2024-09-17T01:13:32.100 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:32 smithi033 ceph-mon[65430]: fsmap foofs:1 {0=foofs.smithi100.ssfebu=up:active} 1 up:standby 2024-09-17T01:13:32.100 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:32 smithi033 ceph-mon[65430]: pgmap v299: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 95 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 1.6 MiB/s wr, 212 op/s; 23/1599 objects degraded (1.438%); 3.3 MiB/s, 46 objects/s recovering 2024-09-17T01:13:32.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 226933 8.32 MB/sec execute 571 sec latency 89.834 ms 2024-09-17T01:13:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:32 smithi100 ceph-mon[51701]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T01:13:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:32 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.100:6832/4118010065,v1:172.21.15.100:6833/4118010065] up:active 2024-09-17T01:13:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:32 smithi100 ceph-mon[51701]: fsmap foofs:1 {0=foofs.smithi100.ssfebu=up:active} 1 up:standby 2024-09-17T01:13:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:32 smithi100 ceph-mon[51701]: pgmap v299: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 95 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 30 MiB/s rd, 1.6 MiB/s wr, 212 op/s; 23/1599 objects degraded (1.438%); 3.3 MiB/s, 46 objects/s recovering 2024-09-17T01:13:33.131 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:33 smithi033 ceph-mon[65430]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1599 objects degraded (1.438%), 1 pg degraded) 2024-09-17T01:13:33.131 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:33 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:13:33.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:33 smithi100 ceph-mon[51701]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 23/1599 objects degraded (1.438%), 1 pg degraded) 2024-09-17T01:13:33.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:33 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:13:33.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227038 8.31 MB/sec execute 572 sec latency 130.381 ms 2024-09-17T01:13:34.191 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:34 smithi033 ceph-mon[65430]: pgmap v300: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.3 MiB/s wr, 181 op/s; 2.8 MiB/s, 42 objects/s recovering 2024-09-17T01:13:34.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227164 8.31 MB/sec execute 573 sec latency 100.216 ms 2024-09-17T01:13:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:34 smithi100 ceph-mon[51701]: pgmap v300: 97 pgs: 97 active+clean; 344 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.3 MiB/s wr, 181 op/s; 2.8 MiB/s, 42 objects/s recovering 2024-09-17T01:13:35.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227254 8.30 MB/sec execute 574 sec latency 169.890 ms 2024-09-17T01:13:36.315 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:36 smithi033 ceph-mon[65430]: pgmap v301: 97 pgs: 97 active+clean; 343 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 653 KiB/s wr, 55 op/s; 1.7 MiB/s, 30 objects/s recovering 2024-09-17T01:13:36.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227365 8.29 MB/sec execute 575 sec latency 128.520 ms 2024-09-17T01:13:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:36 smithi100 ceph-mon[51701]: pgmap v301: 97 pgs: 97 active+clean; 343 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 653 KiB/s wr, 55 op/s; 1.7 MiB/s, 30 objects/s recovering 2024-09-17T01:13:37.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227525 8.28 MB/sec execute 576 sec latency 162.665 ms 2024-09-17T01:13:38.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227644 8.27 MB/sec execute 577 sec latency 107.167 ms 2024-09-17T01:13:38.396 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:38 smithi033 ceph-mon[65430]: pgmap v302: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 938 KiB/s wr, 95 op/s; 1.4 MiB/s, 24 objects/s recovering 2024-09-17T01:13:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:38 smithi100 ceph-mon[51701]: pgmap v302: 97 pgs: 97 active+clean; 345 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 938 KiB/s wr, 95 op/s; 1.4 MiB/s, 24 objects/s recovering 2024-09-17T01:13:39.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:39 smithi033 ceph-mon[65430]: daemon mds.foofs.smithi033.gofagb restarted 2024-09-17T01:13:39.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:39.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:39 smithi033 ceph-mon[65430]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T01:13:39.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:39.172 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:39 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:39.276 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:13:39.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227773 8.26 MB/sec execute 578 sec latency 124.748 ms 2024-09-17T01:13:39.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:39 smithi100 ceph-mon[51701]: daemon mds.foofs.smithi033.gofagb restarted 2024-09-17T01:13:39.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:39 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:39.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:39 smithi100 ceph-mon[51701]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T01:13:39.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:39 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:39.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:39 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:40.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:40 smithi033 ceph-mon[65430]: pgmap v304: 97 pgs: 97 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 154 op/s; 252 KiB/s, 6 objects/s recovering 2024-09-17T01:13:40.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:40 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:boot 2024-09-17T01:13:40.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:40 smithi033 ceph-mon[65430]: fsmap foofs:1 {0=foofs.smithi100.ssfebu=up:active} 1 up:standby 2024-09-17T01:13:40.190 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:40 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:13:40.191 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:40 smithi033 ceph-mon[65430]: from='client.44353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:40.191 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:40 smithi033 ceph-mon[65430]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (9m) 2m ago 18m 26.4M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (6m) 2m ago 18m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (5m) 25s ago 17m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (9m) 2m ago 17m 53.2M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 starting - - - - 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (12m) 25s ago 12m 15.2M - 17.2.0 e1d6a67b021e 46b1d5b48d93 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (8m) 2m ago 19m 504M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:13:40.218 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (7m) 25s ago 17m 443M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (7m) 2m ago 19m 50.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (6m) 25s ago 17m 44.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (11m) 2m ago 11m 274M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (11m) 25s ago 11m 88.7M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (9m) 2m ago 17m 15.0M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (8m) 25s ago 17m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (5m) 2m ago 16m 331M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (4m) 2m ago 16m 269M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (3m) 2m ago 15m 188M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (3m) 2m ago 15m 14.5M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (2m) 25s ago 14m 325M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (103s) 25s ago 14m 271M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (65s) 25s ago 14m 238M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 82283e99afeb 2024-09-17T01:13:40.219 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (29s) 25s ago 13m 12.0M 2703M 19.1.1-330-gc2ddf972 ebb0615149de edf5ede3d097 2024-09-17T01:13:40.220 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (7m) 2m ago 17m 62.7M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:13:40.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 227977 8.25 MB/sec execute 579 sec latency 88.014 ms 2024-09-17T01:13:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:40 smithi100 ceph-mon[51701]: pgmap v304: 97 pgs: 97 active+clean; 348 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.4 MiB/s wr, 154 op/s; 252 KiB/s, 6 objects/s recovering 2024-09-17T01:13:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:40 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:boot 2024-09-17T01:13:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:40 smithi100 ceph-mon[51701]: fsmap foofs:1 {0=foofs.smithi100.ssfebu=up:active} 1 up:standby 2024-09-17T01:13:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:40 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:13:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:40 smithi100 ceph-mon[51701]: from='client.44353 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:40.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:40 smithi100 ceph-mon[51701]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:40.820 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 13 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:13:40.821 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:13:41.197 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:41 smithi033 ceph-mon[65430]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:41.197 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:41 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/1537911779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:41.273 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "osd", 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "crash" 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:13:41.274 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "14/23 daemons upgraded", 2024-09-17T01:13:41.275 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T01:13:41.275 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:13:41.275 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:13:41.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 228262 8.26 MB/sec execute 580 sec latency 81.811 ms 2024-09-17T01:13:41.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:41 smithi100 ceph-mon[51701]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:41.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:41 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/1537911779' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:41.863 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:13:42.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:42 smithi033 ceph-mon[65430]: pgmap v305: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 846 KiB/s rd, 1.5 MiB/s wr, 157 op/s; 12 KiB/s, 2 objects/s recovering 2024-09-17T01:13:42.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:42 smithi033 ceph-mon[65430]: from='client.44365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:42.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:42 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3551376872' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:13:42.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:42.215 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:42.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 228555 8.26 MB/sec execute 581 sec latency 66.115 ms 2024-09-17T01:13:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:42 smithi100 ceph-mon[51701]: pgmap v305: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 846 KiB/s rd, 1.5 MiB/s wr, 157 op/s; 12 KiB/s, 2 objects/s recovering 2024-09-17T01:13:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:42 smithi100 ceph-mon[51701]: from='client.44365 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:13:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:42 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3551376872' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:13:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:42.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:43.244 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:43 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:43.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 229020 8.26 MB/sec execute 582 sec latency 40.314 ms 2024-09-17T01:13:43.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:43 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:44.264 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:44 smithi033 ceph-mon[65430]: pgmap v306: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 1.6 MiB/s wr, 160 op/s 2024-09-17T01:13:44.365 INFO:teuthology.orchestra.run.smithi033.stdout: 5 229434 8.26 MB/sec execute 583 sec latency 42.967 ms 2024-09-17T01:13:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:44 smithi100 ceph-mon[51701]: pgmap v306: 97 pgs: 97 active+clean; 344 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 1.6 MiB/s wr, 160 op/s 2024-09-17T01:13:45.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 229889 8.27 MB/sec execute 584 sec latency 57.933 ms 2024-09-17T01:13:46.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:46 smithi033 ceph-mon[65430]: pgmap v307: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.8 MiB/s wr, 322 op/s 2024-09-17T01:13:46.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:46 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:46.317 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:46 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:46.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 230427 8.28 MB/sec execute 585 sec latency 33.508 ms 2024-09-17T01:13:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:46 smithi100 ceph-mon[51701]: pgmap v307: 97 pgs: 97 active+clean; 349 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.8 MiB/s wr, 322 op/s 2024-09-17T01:13:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:46 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:46 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:47.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 230879 8.30 MB/sec execute 586 sec latency 45.539 ms 2024-09-17T01:13:48.361 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:48 smithi033 ceph-mon[65430]: pgmap v308: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.1 MiB/s wr, 317 op/s 2024-09-17T01:13:48.361 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:48.361 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:48 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:48.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231092 8.29 MB/sec execute 587 sec latency 80.595 ms 2024-09-17T01:13:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:48 smithi100 ceph-mon[51701]: pgmap v308: 97 pgs: 97 active+clean; 346 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 3.1 MiB/s wr, 317 op/s 2024-09-17T01:13:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:48 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:49.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231202 8.28 MB/sec execute 588 sec latency 126.733 ms 2024-09-17T01:13:50.163 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:50 smithi033 ceph-mon[65430]: pgmap v309: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 440 op/s 2024-09-17T01:13:50.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231335 8.27 MB/sec execute 589 sec latency 96.177 ms 2024-09-17T01:13:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:50 smithi100 ceph-mon[51701]: pgmap v309: 97 pgs: 97 active+clean; 348 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 4.3 MiB/s wr, 440 op/s 2024-09-17T01:13:51.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231470 8.27 MB/sec execute 590 sec latency 98.861 ms 2024-09-17T01:13:52.226 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:52 smithi033 ceph-mon[65430]: pgmap v310: 97 pgs: 97 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 4.3 MiB/s wr, 443 op/s 2024-09-17T01:13:52.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231619 8.26 MB/sec execute 591 sec latency 138.097 ms 2024-09-17T01:13:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:52 smithi100 ceph-mon[51701]: pgmap v310: 97 pgs: 97 active+clean; 353 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 4.3 MiB/s wr, 443 op/s 2024-09-17T01:13:53.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231756 8.25 MB/sec execute 592 sec latency 81.696 ms 2024-09-17T01:13:53.552 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: Detected new or changed devices on smithi033 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.553 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi100.ssfebu"]}]: dispatch 2024-09-17T01:13:53.554 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:53 smithi033 ceph-mon[65430]: Upgrade: It appears safe to stop mds.foofs.smithi100.ssfebu 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: Detected new or changed devices on smithi033 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:13:53.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi100.ssfebu"]}]: dispatch 2024-09-17T01:13:53.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:53 smithi100 ceph-mon[51701]: Upgrade: It appears safe to stop mds.foofs.smithi100.ssfebu 2024-09-17T01:13:54.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 231945 8.25 MB/sec execute 593 sec latency 112.187 ms 2024-09-17T01:13:54.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:54 smithi033 ceph-mon[65430]: pgmap v311: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 4.4 MiB/s wr, 447 op/s 2024-09-17T01:13:54.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:54 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:54.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:54 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:13:54.601 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:54 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:54.881 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:54 smithi100 ceph-mon[51701]: pgmap v311: 97 pgs: 97 active+clean; 353 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 4.4 MiB/s wr, 447 op/s 2024-09-17T01:13:54.881 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:54 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:54.881 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:54 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi100.ssfebu", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T01:13:54.881 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:54 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:13:55.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 232113 8.24 MB/sec execute 594 sec latency 77.236 ms 2024-09-17T01:13:55.648 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:55 smithi033 ceph-mon[65430]: Upgrade: Updating mds.foofs.smithi100.ssfebu 2024-09-17T01:13:55.648 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:55 smithi033 ceph-mon[65430]: Deploying daemon mds.foofs.smithi100.ssfebu on smithi100 2024-09-17T01:13:55.670 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:55 smithi100 ceph-mon[51701]: Upgrade: Updating mds.foofs.smithi100.ssfebu 2024-09-17T01:13:55.670 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:55 smithi100 ceph-mon[51701]: Deploying daemon mds.foofs.smithi100.ssfebu on smithi100 2024-09-17T01:13:56.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 232282 8.23 MB/sec execute 595 sec latency 161.469 ms 2024-09-17T01:13:56.577 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[65426]: 2024-09-17T01:13:56.233+0000 7fe2bf6d3640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:13:56.577 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:13:56.303+0000 7f4c0fdba640 -1 bdev(0x559045368c00 /var/lib/ceph/osd/ceph-2/block) aio_submit retries 1 2024-09-17T01:13:56.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: pgmap v312: 97 pgs: 97 active+clean; 347 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.8 MiB/s wr, 485 op/s 2024-09-17T01:13:56.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T01:13:56.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:13:56.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T01:13:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: Standby daemon mds.foofs.smithi033.gofagb assigned to filesystem foofs as rank 0 2024-09-17T01:13:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T01:13:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T01:13:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T01:13:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:56 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:replay} 2024-09-17T01:13:56.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: pgmap v312: 97 pgs: 97 active+clean; 347 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 4.8 MiB/s wr, 485 op/s 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: Standby daemon mds.foofs.smithi033.gofagb assigned to filesystem foofs as rank 0 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T01:13:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:56 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:replay} 2024-09-17T01:13:57.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 232282 8.22 MB/sec execute 596 sec latency 1161.536 ms 2024-09-17T01:13:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:58 smithi033 ceph-mon[65430]: pgmap v314: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.2 MiB/s wr, 449 op/s 2024-09-17T01:13:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:13:58.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 232282 8.20 MB/sec execute 597 sec latency 2161.565 ms 2024-09-17T01:13:58.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:13:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:13:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:58 smithi100 ceph-mon[51701]: pgmap v314: 97 pgs: 97 active+clean; 348 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.8 KiB/s rd, 4.2 MiB/s wr, 449 op/s 2024-09-17T01:13:58.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:13:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:59 smithi033 ceph-mon[65430]: reconnect by client.14540 172.21.15.33:0/2965077002 after 0 2024-09-17T01:13:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:59 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:reconnect 2024-09-17T01:13:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:59 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:reconnect} 2024-09-17T01:13:59.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:13:59 smithi033 ceph-mon[65430]: reconnect by client.24361 172.21.15.100:0/3113584054 after 0.0359994 2024-09-17T01:13:59.366 INFO:teuthology.orchestra.run.smithi033.stdout: 5 232282 8.19 MB/sec execute 598 sec latency 3161.592 ms 2024-09-17T01:13:59.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:59 smithi100 ceph-mon[51701]: reconnect by client.14540 172.21.15.33:0/2965077002 after 0 2024-09-17T01:13:59.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:59 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:reconnect 2024-09-17T01:13:59.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:59 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:reconnect} 2024-09-17T01:13:59.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:13:59 smithi100 ceph-mon[51701]: reconnect by client.24361 172.21.15.100:0/3113584054 after 0.0359994 2024-09-17T01:14:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:00 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:rejoin 2024-09-17T01:14:00.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:00 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:rejoin} 2024-09-17T01:14:00.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:00 smithi033 ceph-mon[65430]: pgmap v315: 97 pgs: 97 active+clean; 345 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.7 MiB/s wr, 261 op/s 2024-09-17T01:14:00.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 232282 8.18 MB/sec execute 599 sec latency 4161.618 ms 2024-09-17T01:14:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:00 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:rejoin 2024-09-17T01:14:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:00 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:rejoin} 2024-09-17T01:14:00.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:00 smithi100 ceph-mon[51701]: pgmap v315: 97 pgs: 97 active+clean; 345 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.7 MiB/s wr, 261 op/s 2024-09-17T01:14:01.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:01 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:clientreplay 2024-09-17T01:14:01.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:01 smithi033 ceph-mon[65430]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:clientreplay} 2024-09-17T01:14:01.289 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:01 smithi033 ceph-mon[65430]: daemon mds.foofs.smithi033.gofagb is now active in filesystem foofs as rank 0 2024-09-17T01:14:01.367 INFO:teuthology.orchestra.run.smithi033.stdout: 5 cleanup 600 sec 2024-09-17T01:14:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:01 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:clientreplay 2024-09-17T01:14:01.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:01 smithi100 ceph-mon[51701]: fsmap foofs:1/1 {0=foofs.smithi033.gofagb=up:clientreplay} 2024-09-17T01:14:01.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:01 smithi100 ceph-mon[51701]: daemon mds.foofs.smithi033.gofagb is now active in filesystem foofs as rank 0 2024-09-17T01:14:01.629 INFO:teuthology.orchestra.run.smithi033.stdout: 0 cleanup 600 sec 2024-09-17T01:14:01.629 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:14:01.629 INFO:teuthology.orchestra.run.smithi033.stdout: Operation Count AvgLat MaxLat 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: ---------------------------------------- 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: NTCreateX 157313 5.502 6878.748 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Close 115561 0.141 21.507 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Rename 6674 10.642 1057.346 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Unlink 31775 7.996 2367.878 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Qpathinfo 142717 1.545 6839.125 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Qfileinfo 24874 0.007 0.985 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Qfsinfo 26145 0.708 50.638 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Sfileinfo 12848 14.600 1029.703 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: Find 55138 1.005 64.809 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: WriteX 77741 16.787 6866.269 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: ReadX 246712 0.010 10.995 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: LockX 512 0.424 17.355 2024-09-17T01:14:01.630 INFO:teuthology.orchestra.run.smithi033.stdout: UnlockX 512 0.389 15.132 2024-09-17T01:14:01.631 INFO:teuthology.orchestra.run.smithi033.stdout: Flush 11037 0.003 0.081 2024-09-17T01:14:01.631 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:14:01.631 INFO:teuthology.orchestra.run.smithi033.stdout:Throughput 8.17551 MB/sec 5 clients 5 procs max_latency=6878.756 ms 2024-09-17T01:14:01.632 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-17T01:14:01.671 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-17T01:14:01.799 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-17T01:14:01.825 INFO:teuthology.orchestra.run.smithi033.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T01:14:01.825 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:14:01.825 INFO:teuthology.orchestra.run.smithi033.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-17T01:14:01.825 INFO:teuthology.orchestra.run.smithi033.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-17T01:14:01.926 INFO:teuthology.orchestra.run.smithi033.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T01:14:01.926 INFO:teuthology.orchestra.run.smithi033.stdout:releasing clients 2024-09-17T01:14:02.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:02 smithi033 ceph-mon[65430]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T01:14:02.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:02 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:active 2024-09-17T01:14:02.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:02 smithi033 ceph-mon[65430]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 2024-09-17T01:14:02.050 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:02 smithi033 ceph-mon[65430]: pgmap v316: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 192 op/s 2024-09-17T01:14:02.069 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:02 smithi100 ceph-mon[51701]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T01:14:02.069 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:02 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982] up:active 2024-09-17T01:14:02.069 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:02 smithi100 ceph-mon[51701]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 2024-09-17T01:14:02.069 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:02 smithi100 ceph-mon[51701]: pgmap v316: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 1.9 MiB/s wr, 192 op/s 2024-09-17T01:14:03.926 INFO:teuthology.orchestra.run.smithi033.stdout: 5 311 15.35 MB/sec execute 1 sec latency 145.372 ms 2024-09-17T01:14:04.135 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:04.135 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:04.135 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:04.135 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T01:14:04.135 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: Cluster is now healthy 2024-09-17T01:14:04.135 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: mds.? [v2:172.21.15.100:6832/1044354364,v1:172.21.15.100:6833/1044354364] up:boot 2024-09-17T01:14:04.136 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:14:04.136 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:14:04.136 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:04 smithi100 ceph-mon[51701]: pgmap v317: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.7 MiB/s wr, 193 op/s 2024-09-17T01:14:04.206 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:04.206 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: Cluster is now healthy 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: mds.? [v2:172.21.15.100:6832/1044354364,v1:172.21.15.100:6833/1044354364] up:boot 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: fsmap foofs:1 {0=foofs.smithi033.gofagb=up:active} 1 up:standby 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:14:04.207 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:04 smithi033 ceph-mon[65430]: pgmap v317: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.7 MiB/s wr, 193 op/s 2024-09-17T01:14:04.926 INFO:teuthology.orchestra.run.smithi033.stdout: 5 452 18.99 MB/sec execute 2 sec latency 59.170 ms 2024-09-17T01:14:05.926 INFO:teuthology.orchestra.run.smithi033.stdout: 5 627 20.57 MB/sec execute 3 sec latency 139.460 ms 2024-09-17T01:14:06.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:06 smithi100 ceph-mon[51701]: pgmap v318: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.2 MiB/s wr, 223 op/s 2024-09-17T01:14:06.311 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:06 smithi033 ceph-mon[65430]: pgmap v318: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 4.2 MiB/s wr, 223 op/s 2024-09-17T01:14:06.926 INFO:teuthology.orchestra.run.smithi033.stdout: 5 885 16.70 MB/sec execute 4 sec latency 162.497 ms 2024-09-17T01:14:07.325 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.325 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.325 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.325 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:07 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.560 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:07 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:07.926 INFO:teuthology.orchestra.run.smithi033.stdout: 5 cleanup 5 sec 2024-09-17T01:14:08.186 INFO:teuthology.orchestra.run.smithi033.stdout: 0 cleanup 5 sec 2024-09-17T01:14:08.186 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:14:08.186 INFO:teuthology.orchestra.run.smithi033.stdout: Operation Count AvgLat MaxLat 2024-09-17T01:14:08.186 INFO:teuthology.orchestra.run.smithi033.stdout: ---------------------------------------- 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: NTCreateX 701 9.660 145.367 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Close 651 10.796 162.493 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Rename 41 5.481 31.355 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Unlink 65 9.774 35.258 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Qpathinfo 360 1.289 25.962 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Qfileinfo 328 18.791 136.387 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Qfsinfo 141 0.480 2.627 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Sfileinfo 134 14.603 91.900 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Find 177 0.981 11.437 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: WriteX 1601 0.026 0.072 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: ReadX 614 0.021 0.606 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: Flush 120 14.924 43.389 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:14:08.187 INFO:teuthology.orchestra.run.smithi033.stdout:Throughput 16.7045 MB/sec 5 clients 5 procs max_latency=162.497 ms 2024-09-17T01:14:08.189 DEBUG:teuthology.parallel:result is None 2024-09-17T01:14:08.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:08 smithi033 ceph-mon[65430]: pgmap v319: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 7.8 MiB/s wr, 264 op/s 2024-09-17T01:14:08.625 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:08 smithi100 ceph-mon[51701]: pgmap v319: 97 pgs: 97 active+clean; 333 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 7.8 MiB/s wr, 264 op/s 2024-09-17T01:14:10.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:10 smithi033 ceph-mon[65430]: pgmap v320: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 9.6 MiB/s wr, 246 op/s 2024-09-17T01:14:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:10 smithi100 ceph-mon[51701]: pgmap v320: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 9.6 MiB/s wr, 246 op/s 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: Detected new or changed devices on smithi100 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all mds 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi033.gofagb"}]': finished 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi100.ssfebu"}]': finished 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all rgw 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:14:11.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all iscsi 2024-09-17T01:14:11.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:11 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: Detected new or changed devices on smithi100 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all mds 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi033.gofagb"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi033.gofagb"}]': finished 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi100.ssfebu"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi100.ssfebu"}]': finished 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all rgw 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T01:14:11.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all iscsi 2024-09-17T01:14:11.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:11 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:12.254 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: pgmap v321: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 9.6 MiB/s wr, 241 op/s 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:14:12.881 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:12.882 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:14:12.882 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:14:12.882 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:14:12.882 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:12.967 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:14:12.967 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (9m) 26s ago 18m 26.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:14:12.967 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (6m) 26s ago 18m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:14:12.967 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (6m) 6s ago 18m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (9m) 26s ago 18m 53.3M - 8.3.5 dad864ee21e9 874d20628a28 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (35s) 26s ago 12m 11.4M - 19.1.1-330-gc2ddf972 ebb0615149de 3ca4e3bd7822 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (10s) 6s ago 12m 15.1M - 19.1.1-330-gc2ddf972 ebb0615149de 9e5b635bef27 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (9m) 26s ago 19m 509M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (8m) 6s ago 17m 444M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (7m) 26s ago 19m 58.7M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (7m) 6s ago 17m 47.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (12m) 26s ago 12m 278M - 4.0 e1d6a67b021e 73ecd5d749dd 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (12m) 6s ago 12m 88.8M - 4.0 e1d6a67b021e 83fecdfebef4 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (9m) 26s ago 18m 16.6M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (9m) 6s ago 17m 16.3M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (5m) 26s ago 17m 400M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:14:12.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (5m) 26s ago 16m 321M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (4m) 26s ago 16m 340M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (3m) 26s ago 15m 332M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (2m) 6s ago 15m 353M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (2m) 6s ago 15m 309M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (98s) 6s ago 14m 275M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 82283e99afeb 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (62s) 6s ago 14m 314M 2703M 19.1.1-330-gc2ddf972 ebb0615149de edf5ede3d097 2024-09-17T01:14:12.969 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (8m) 26s ago 17m 67.5M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:14:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: pgmap v321: 97 pgs: 97 active+clean; 331 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 9.6 MiB/s wr, 241 op/s 2024-09-17T01:14:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:14:12.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:14:12.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:14:12.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:12.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:14:12.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:14:12.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi033.ttskqh-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:14:12.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:13.401 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:14:13.401 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:14:13.401 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:14:13.401 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:13.401 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:14:13.401 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:14:13.402 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:14:13.767 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:14:13.767 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:14:13.767 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "osd", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "crash", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "mds" 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "16/23 daemons upgraded", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading nfs daemons", 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:14:13.768 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Upgrade: Updating nfs.foo.0.0.smithi033.ttskqh (1/2) 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Rados config object exists: conf-nfs.foo 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh-rgw 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Bind address in nfs.foo.0.0.smithi033.ttskqh's ganesha conf is defaulting to empty 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: Deploying daemon nfs.foo.0.0.smithi033.ttskqh on smithi033 2024-09-17T01:14:13.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:13.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:13 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2260910240' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: from='client.34494 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Upgrade: Updating nfs.foo.0.0.smithi033.ttskqh (1/2) 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Rados config object exists: conf-nfs.foo 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Creating key for client.nfs.foo.0.0.smithi033.ttskqh-rgw 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Bind address in nfs.foo.0.0.smithi033.ttskqh's ganesha conf is defaulting to empty 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: Deploying daemon nfs.foo.0.0.smithi033.ttskqh on smithi033 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: from='client.34498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:14.008 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:13 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2260910240' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:14.226 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:14:14.921 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:14 smithi033 ceph-mon[65430]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:14.921 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:14 smithi033 ceph-mon[65430]: pgmap v322: 97 pgs: 97 active+clean; 308 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 10 MiB/s wr, 289 op/s 2024-09-17T01:14:14.921 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:14 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3294602436' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:14:14.953 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:14 smithi100 ceph-mon[51701]: from='client.34512 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:14.953 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:14 smithi100 ceph-mon[51701]: pgmap v322: 97 pgs: 97 active+clean; 308 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 10 MiB/s wr, 289 op/s 2024-09-17T01:14:14.953 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:14 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3294602436' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:14:15.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:15 smithi100 ceph-mon[51701]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:16.079 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:15 smithi033 ceph-mon[65430]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:16.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:16 smithi100 ceph-mon[51701]: pgmap v323: 97 pgs: 97 active+clean; 252 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 9.9 MiB/s wr, 291 op/s 2024-09-17T01:14:16.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:16 smithi100 ceph-mon[51701]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.33:0/2965077002"}]: dispatch 2024-09-17T01:14:16.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:16.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:16 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:16.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:16 smithi033 ceph-mon[65430]: pgmap v323: 97 pgs: 97 active+clean; 252 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 KiB/s rd, 9.9 MiB/s wr, 291 op/s 2024-09-17T01:14:16.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:16 smithi033 ceph-mon[65430]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.33:0/2965077002"}]: dispatch 2024-09-17T01:14:16.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:16.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:16 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:17.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:17 smithi100 ceph-mon[51701]: Evicting (and blocklisting) client session 14540 (172.21.15.33:0/2965077002) 2024-09-17T01:14:17.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:17 smithi100 ceph-mon[51701]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.33:0/2965077002"}]': finished 2024-09-17T01:14:17.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:17 smithi100 ceph-mon[51701]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T01:14:18.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:17 smithi033 ceph-mon[65430]: Evicting (and blocklisting) client session 14540 (172.21.15.33:0/2965077002) 2024-09-17T01:14:18.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:17 smithi033 ceph-mon[65430]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.33:0/2965077002"}]': finished 2024-09-17T01:14:18.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:17 smithi033 ceph-mon[65430]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T01:14:18.951 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 198 op/s 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:14:18.952 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:18 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: pgmap v325: 97 pgs: 97 active+clean; 232 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.3 MiB/s wr, 198 op/s 2024-09-17T01:14:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T01:14:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T01:14:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T01:14:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:19.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T01:14:19.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T01:14:19.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi100.mvigkp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T01:14:19.060 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:18 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Upgrade: Updating nfs.foo.1.0.smithi100.mvigkp (2/2) 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Rados config object exists: conf-nfs.foo 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp-rgw 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Bind address in nfs.foo.1.0.smithi100.mvigkp's ganesha conf is defaulting to empty 2024-09-17T01:14:19.797 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:19 smithi100 ceph-mon[51701]: Deploying daemon nfs.foo.1.0.smithi100.mvigkp on smithi100 2024-09-17T01:14:20.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Upgrade: Updating nfs.foo.1.0.smithi100.mvigkp (2/2) 2024-09-17T01:14:20.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp 2024-09-17T01:14:20.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T01:14:20.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Rados config object exists: conf-nfs.foo 2024-09-17T01:14:20.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Creating key for client.nfs.foo.1.0.smithi100.mvigkp-rgw 2024-09-17T01:14:20.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Bind address in nfs.foo.1.0.smithi100.mvigkp's ganesha conf is defaulting to empty 2024-09-17T01:14:20.114 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:19 smithi033 ceph-mon[65430]: Deploying daemon nfs.foo.1.0.smithi100.mvigkp on smithi100 2024-09-17T01:14:20.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:20 smithi033 ceph-mon[65430]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 873 KiB/s wr, 106 op/s 2024-09-17T01:14:20.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:20 smithi100 ceph-mon[51701]: pgmap v326: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 873 KiB/s wr, 106 op/s 2024-09-17T01:14:22.941 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:22 smithi033 ceph-mon[65430]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 499 KiB/s wr, 93 op/s 2024-09-17T01:14:22.942 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:22.942 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:22.942 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:22 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:22.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:22 smithi100 ceph-mon[51701]: pgmap v327: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 499 KiB/s wr, 93 op/s 2024-09-17T01:14:22.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:22.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:22.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:22 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:24.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:24 smithi033 ceph-mon[65430]: pgmap v328: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 28 KiB/s wr, 28 op/s 2024-09-17T01:14:24.933 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:24 smithi100 ceph-mon[51701]: pgmap v328: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 28 KiB/s wr, 28 op/s 2024-09-17T01:14:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: pgmap v329: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 26 KiB/s wr, 14 op/s 2024-09-17T01:14:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: Evicting (and blocklisting) client session 24361 (172.21.15.100:0/3113584054) 2024-09-17T01:14:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.100:0/3113584054"}]: dispatch 2024-09-17T01:14:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:26 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: pgmap v329: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.6 KiB/s rd, 26 KiB/s wr, 14 op/s 2024-09-17T01:14:26.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: Evicting (and blocklisting) client session 24361 (172.21.15.100:0/3113584054) 2024-09-17T01:14:26.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.100:0/3113584054"}]: dispatch 2024-09-17T01:14:26.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:26.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:26 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:27.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:27 smithi100 ceph-mon[51701]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.100:0/3113584054"}]': finished 2024-09-17T01:14:27.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:27 smithi100 ceph-mon[51701]: osdmap e160: 8 total, 8 up, 8 in 2024-09-17T01:14:27.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:27.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:27 smithi033 ceph-mon[65430]: from='mds.? [v2:172.21.15.33:6834/433981982,v1:172.21.15.33:6835/433981982]' entity='mds.foofs.smithi033.gofagb' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.100:0/3113584054"}]': finished 2024-09-17T01:14:27.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:27 smithi033 ceph-mon[65430]: osdmap e160: 8 total, 8 up, 8 in 2024-09-17T01:14:27.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:28 smithi100 ceph-mon[51701]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 KiB/s wr, 7 op/s 2024-09-17T01:14:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:28 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.542 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:28 smithi033 ceph-mon[65430]: pgmap v331: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 KiB/s wr, 7 op/s 2024-09-17T01:14:28.542 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.542 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.542 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:28.542 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:28 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:30 smithi033 ceph-mon[65430]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-09-17T01:14:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:30.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:30 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:30 smithi100 ceph-mon[51701]: pgmap v332: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-09-17T01:14:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:30.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:30 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: pgmap v333: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:32.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all nfs 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi033.ttskqh"}]: dispatch 2024-09-17T01:14:32.615 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi033.ttskqh"}]': finished 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi100.mvigkp"}]: dispatch 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi100.mvigkp"}]': finished 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.616 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:32 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: pgmap v333: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.9 KiB/s wr, 4 op/s 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.729 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all nfs 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi033.ttskqh"}]: dispatch 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi033.ttskqh"}]': finished 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi100.mvigkp"}]: dispatch 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi100.mvigkp"}]': finished 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: Upgrade: Setting container_image for all nvmeof 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:32.730 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:32 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:34.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:34 smithi100 ceph-mon[51701]: Upgrade: Updating grafana.smithi033 2024-09-17T01:14:34.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:34 smithi100 ceph-mon[51701]: Deploying daemon grafana.smithi033 on smithi033 2024-09-17T01:14:34.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:34 smithi100 ceph-mon[51701]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-17T01:14:34.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:34 smithi033 ceph-mon[65430]: Upgrade: Updating grafana.smithi033 2024-09-17T01:14:34.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:34 smithi033 ceph-mon[65430]: Deploying daemon grafana.smithi033 on smithi033 2024-09-17T01:14:34.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:34 smithi033 ceph-mon[65430]: pgmap v334: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 1.4 KiB/s wr, 4 op/s 2024-09-17T01:14:36.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:36 smithi100 ceph-mon[51701]: pgmap v335: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 614 B/s wr, 1 op/s 2024-09-17T01:14:36.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:36 smithi033 ceph-mon[65430]: pgmap v335: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 614 B/s wr, 1 op/s 2024-09-17T01:14:38.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:38 smithi100 ceph-mon[51701]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 B/s rd, 468 B/s wr, 0 op/s 2024-09-17T01:14:38.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:38 smithi033 ceph-mon[65430]: pgmap v336: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 93 B/s rd, 468 B/s wr, 0 op/s 2024-09-17T01:14:40.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:40 smithi033 ceph-mon[65430]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-17T01:14:40.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:40 smithi100 ceph-mon[51701]: pgmap v337: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-17T01:14:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:42 smithi100 ceph-mon[51701]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:42.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:42.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:42 smithi033 ceph-mon[65430]: pgmap v338: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:42.789 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:44.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:44 smithi033 ceph-mon[65430]: pgmap v339: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:44.621 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:14:44.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:44 smithi100 ceph-mon[51701]: pgmap v339: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:45.351 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:14:45.351 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (10m) 18s ago 19m 26.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:14:45.351 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (7m) 18s ago 19m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:14:45.351 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (7m) 20s ago 18m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 starting - - - - 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (67s) 18s ago 13m 40.7M - 19.1.1-330-gc2ddf972 ebb0615149de 3ca4e3bd7822 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (42s) 20s ago 13m 15.2M - 19.1.1-330-gc2ddf972 ebb0615149de 9e5b635bef27 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (9m) 18s ago 20m 510M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (8m) 20s ago 18m 444M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (8m) 18s ago 20m 62.0M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (8m) 20s ago 18m 48.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (29s) 18s ago 12m 57.7M - 5.5 ebb0615149de 1357e68bff5a 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (23s) 20s ago 12m 15.4M - 5.5 ebb0615149de 492568f157ac 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (10m) 18s ago 18m 16.9M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (9m) 20s ago 18m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (6m) 18s ago 17m 421M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:14:45.352 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (5m) 18s ago 17m 343M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (4m) 18s ago 16m 264M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (4m) 18s ago 16m 360M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (3m) 20s ago 15m 361M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (2m) 20s ago 15m 309M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (2m) 20s ago 15m 280M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 82283e99afeb 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (94s) 20s ago 14m 318M 2703M 19.1.1-330-gc2ddf972 ebb0615149de edf5ede3d097 2024-09-17T01:14:45.353 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (8m) 18s ago 18m 67.6M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:14:45.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:45 smithi033 ceph-mon[65430]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:45.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:45 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:45.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:45 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:45.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:45 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:45 smithi100 ceph-mon[51701]: from='client.34562 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:45 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:45.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:45 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:45.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:45 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:45.826 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:14:45.826 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:14:45.826 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:14:45.826 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:14:45.827 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:14:45.828 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:14:45.828 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T01:14:45.828 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:14:45.828 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "osd", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "crash", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "nfs", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr", 2024-09-17T01:14:46.202 INFO:teuthology.orchestra.run.smithi033.stdout: "mds" 2024-09-17T01:14:46.203 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:14:46.203 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "18/23 daemons upgraded", 2024-09-17T01:14:46.203 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading grafana daemons", 2024-09-17T01:14:46.203 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:14:46.203 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:14:46.715 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:14:46.715 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:46 smithi033 ceph-mon[65430]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:46.715 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:46 smithi033 ceph-mon[65430]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:46.715 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:46 smithi033 ceph-mon[65430]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:46.715 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:46 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/496585065' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:46.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:46 smithi100 ceph-mon[51701]: from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:46.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:46 smithi100 ceph-mon[51701]: pgmap v340: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:46.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:46 smithi100 ceph-mon[51701]: from='client.34570 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:46.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:46 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/496585065' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:47.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:47 smithi100 ceph-mon[51701]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:47.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:47 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/784168491' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:14:47.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:47 smithi033 ceph-mon[65430]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:14:47.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:47 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/784168491' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:14:48.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:48 smithi100 ceph-mon[51701]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:48.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:48 smithi033 ceph-mon[65430]: pgmap v341: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:50.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:50 smithi100 ceph-mon[51701]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:14:50.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:50 smithi033 ceph-mon[65430]: pgmap v342: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:14:52.673 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:52 smithi033 ceph-mon[65430]: pgmap v343: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:52.673 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.673 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.674 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.674 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:52 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:52 smithi100 ceph-mon[51701]: pgmap v343: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:52.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:52.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:52 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:54.679 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:54 smithi033 ceph-mon[65430]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:54.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:54 smithi100 ceph-mon[51701]: pgmap v344: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:56.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.866 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T01:14:56.867 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:14:56.868 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:56.869 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.870 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.870 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:56.870 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:56.870 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:56.870 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.870 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: pgmap v345: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.979 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T01:14:56.980 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.981 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T01:14:56.982 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:56.983 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:57.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:57 smithi100 ceph-mon[51701]: Upgrade: Finalizing container_image settings 2024-09-17T01:14:57.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:57 smithi100 ceph-mon[51701]: Upgrade: Complete! 2024-09-17T01:14:57.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:57.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:57 smithi033 ceph-mon[65430]: Upgrade: Finalizing container_image settings 2024-09-17T01:14:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:57 smithi033 ceph-mon[65430]: Upgrade: Complete! 2024-09-17T01:14:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:14:58.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:14:58.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:14:58 smithi100 ceph-mon[51701]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:14:59.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:14:58 smithi033 ceph-mon[65430]: pgmap v346: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:00.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:00 smithi100 ceph-mon[51701]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:01.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:00 smithi033 ceph-mon[65430]: pgmap v347: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:02.978 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:02 smithi100 ceph-mon[51701]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:03.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:02 smithi033 ceph-mon[65430]: pgmap v348: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:04.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:04 smithi100 ceph-mon[51701]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:05.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:04 smithi033 ceph-mon[65430]: pgmap v349: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:06.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:06 smithi100 ceph-mon[51701]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:07.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:06 smithi033 ceph-mon[65430]: pgmap v350: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:08.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:08 smithi100 ceph-mon[51701]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:09.058 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:08 smithi033 ceph-mon[65430]: pgmap v351: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:10.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:10 smithi100 ceph-mon[51701]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:11.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:10 smithi033 ceph-mon[65430]: pgmap v352: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:13.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:12 smithi033 ceph-mon[65430]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:13.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:13.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:12 smithi100 ceph-mon[51701]: pgmap v353: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:13.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:15.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:14 smithi033 ceph-mon[65430]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:15.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:14 smithi100 ceph-mon[51701]: pgmap v354: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:17.086 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:16 smithi033 ceph-mon[65430]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:17.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:16 smithi100 ceph-mon[51701]: pgmap v355: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:17.842 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:15:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:18 smithi033 ceph-mon[65430]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:15:19.059 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:18 smithi033 ceph-mon[65430]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:19.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:18 smithi100 ceph-mon[51701]: from='client.34586 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:15:19.228 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:18 smithi100 ceph-mon[51701]: pgmap v356: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:19.939 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:15:19.939 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (11m) 28s ago 19m 26.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:15:19.939 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (7m) 28s ago 19m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:15:19.939 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (7m) 54s ago 19m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (35s) 28s ago 19m 38.8M - 9.4.12 2bacad6d85d8 0aefdf7a7493 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (101s) 28s ago 13m 40.8M - 19.1.1-330-gc2ddf972 ebb0615149de 3ca4e3bd7822 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (77s) 54s ago 13m 15.2M - 19.1.1-330-gc2ddf972 ebb0615149de 9e5b635bef27 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (10m) 28s ago 20m 517M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (9m) 54s ago 19m 444M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (9m) 28s ago 21m 63.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (8m) 54s ago 18m 48.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (63s) 28s ago 13m 58.1M - 5.5 ebb0615149de 1357e68bff5a 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (58s) 54s ago 13m 15.4M - 5.5 ebb0615149de 492568f157ac 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (10m) 28s ago 19m 16.5M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:15:19.940 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (10m) 54s ago 18m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (7m) 28s ago 18m 421M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (6m) 28s ago 17m 343M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (5m) 28s ago 17m 265M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (4m) 28s ago 16m 360M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (3m) 54s ago 16m 361M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (3m) 54s ago 16m 309M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (2m) 54s ago 15m 280M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 82283e99afeb 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (2m) 54s ago 15m 318M 2703M 19.1.1-330-gc2ddf972 ebb0615149de edf5ede3d097 2024-09-17T01:15:19.941 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (9m) 28s ago 18m 67.6M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:15:20.613 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:15:21.092 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:20 smithi033 ceph-mon[65430]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:21.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:20 smithi100 ceph-mon[51701]: pgmap v357: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:22.015 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:21 smithi033 ceph-mon[65430]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:15:22.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:21 smithi100 ceph-mon[51701]: from='client.34590 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:15:22.879 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:15:22.879 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:15:22.879 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:15:22.879 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:15:22.879 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:15:22.879 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:15:22.880 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:15:22.881 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:15:22.881 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T01:15:22.881 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:15:22.881 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:15:23.113 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:22 smithi033 ceph-mon[65430]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:23.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:22 smithi100 ceph-mon[51701]: pgmap v358: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:23.604 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T01:15:23.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:23 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/124148419' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:15:23.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:23 smithi033 ceph-mon[65430]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:24.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:23 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/124148419' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:15:24.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:23 smithi100 ceph-mon[51701]: pgmap v359: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:25.466 INFO:teuthology.orchestra.run.smithi033.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T01:15:26.161 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 60' 2024-09-17T01:15:26.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:26 smithi033 ceph-mon[65430]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:26 smithi100 ceph-mon[51701]: pgmap v360: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:27.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:27.527 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:28 smithi100 ceph-mon[51701]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:28.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:28 smithi033 ceph-mon[65430]: pgmap v361: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:30 smithi100 ceph-mon[51701]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:30.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:30 smithi033 ceph-mon[65430]: pgmap v362: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:32 smithi100 ceph-mon[51701]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:32.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:32 smithi033 ceph-mon[65430]: pgmap v363: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:34 smithi100 ceph-mon[51701]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:34.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:34 smithi033 ceph-mon[65430]: pgmap v364: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:36.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:36 smithi100 ceph-mon[51701]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:36.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:36 smithi033 ceph-mon[65430]: pgmap v365: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:38 smithi100 ceph-mon[51701]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:38.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:38 smithi033 ceph-mon[65430]: pgmap v366: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:40 smithi100 ceph-mon[51701]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:40.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:40 smithi033 ceph-mon[65430]: pgmap v367: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:42 smithi100 ceph-mon[51701]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:42 smithi033 ceph-mon[65430]: pgmap v368: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:42.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:44 smithi100 ceph-mon[51701]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:44.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:44 smithi033 ceph-mon[65430]: pgmap v369: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:46 smithi100 ceph-mon[51701]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:46.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:46 smithi033 ceph-mon[65430]: pgmap v370: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:48.480 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:48 smithi100 ceph-mon[51701]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:48.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:48 smithi033 ceph-mon[65430]: pgmap v371: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:50 smithi100 ceph-mon[51701]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:50.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:50 smithi033 ceph-mon[65430]: pgmap v372: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:15:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:52 smithi100 ceph-mon[51701]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:52.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:52 smithi033 ceph-mon[65430]: pgmap v373: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:54.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:54 smithi100 ceph-mon[51701]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:54.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:54 smithi033 ceph-mon[65430]: pgmap v374: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:56.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:56 smithi100 ceph-mon[51701]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:56.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:56 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:15:56.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:56 smithi033 ceph-mon[65430]: pgmap v375: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:56.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:56 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:15:57.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:15:57.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:15:57.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:57.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:15:57.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:15:57.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.864 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:15:57.865 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:15:58.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:15:58 smithi100 ceph-mon[51701]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:15:58.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:15:58 smithi033 ceph-mon[65430]: pgmap v376: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:00.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:00 smithi100 ceph-mon[51701]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:00.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:00 smithi033 ceph-mon[65430]: pgmap v377: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:02.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:02 smithi100 ceph-mon[51701]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:02.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:02 smithi033 ceph-mon[65430]: pgmap v378: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:04.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:04 smithi100 ceph-mon[51701]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:04.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:04 smithi033 ceph-mon[65430]: pgmap v379: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:06.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:06 smithi100 ceph-mon[51701]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:06.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:06 smithi033 ceph-mon[65430]: pgmap v380: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:08.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:08 smithi100 ceph-mon[51701]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:08.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:08 smithi033 ceph-mon[65430]: pgmap v381: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:10.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:10 smithi100 ceph-mon[51701]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:10.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:10 smithi033 ceph-mon[65430]: pgmap v382: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:12 smithi100 ceph-mon[51701]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:12.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:12 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:12.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:12 smithi033 ceph-mon[65430]: pgmap v383: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:12.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:12 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:14.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:14 smithi100 ceph-mon[51701]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:14.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:14 smithi033 ceph-mon[65430]: pgmap v384: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:16.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:16 smithi100 ceph-mon[51701]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:16.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:16 smithi033 ceph-mon[65430]: pgmap v385: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:18.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:18 smithi100 ceph-mon[51701]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:18.863 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:18 smithi033 ceph-mon[65430]: pgmap v386: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:20.227 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:19 smithi100 ceph-mon[51701]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:20.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:19 smithi033 ceph-mon[65430]: pgmap v387: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:22.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:22 smithi100 ceph-mon[51701]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:22.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:22 smithi033 ceph-mon[65430]: pgmap v388: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:24.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:24 smithi100 ceph-mon[51701]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:24.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:24 smithi033 ceph-mon[65430]: pgmap v389: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:26.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:26 smithi100 ceph-mon[51701]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:26.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:26 smithi033 ceph-mon[65430]: pgmap v390: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:27.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:27 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:27.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:27 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:28.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:28 smithi100 ceph-mon[51701]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:28.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:28 smithi033 ceph-mon[65430]: pgmap v391: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:28.643 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T01:16:30.192 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:30 smithi033 ceph-mon[65430]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:30.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:30 smithi100 ceph-mon[51701]: pgmap v392: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.smithi033 smithi033 *:9093,9094 running (12m) 99s ago 20m 26.5M - 0.25.0 c8568f914cd2 e292ed591cb4 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi033 smithi033 running (8m) 99s ago 20m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 8caee343c0d9 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:crash.smithi100 smithi100 running (8m) 2m ago 20m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 74f00c9d08db 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.smithi033 smithi033 *:3000 running (106s) 99s ago 20m 38.8M - 9.4.12 2bacad6d85d8 0aefdf7a7493 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi033.gofagb smithi033 running (2m) 99s ago 15m 40.8M - 19.1.1-330-gc2ddf972 ebb0615149de 3ca4e3bd7822 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:mds.foofs.smithi100.ssfebu smithi100 running (2m) 2m ago 15m 15.2M - 19.1.1-330-gc2ddf972 ebb0615149de 9e5b635bef27 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi033.illmog smithi033 *:8443,9283,8765 running (11m) 99s ago 22m 517M - 19.1.1-330-gc2ddf972 ebb0615149de 886de0c291ff 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.smithi100.qrwkie smithi100 *:8443,9283,8765 running (10m) 2m ago 20m 444M - 19.1.1-330-gc2ddf972 ebb0615149de 934e16a0d3e7 2024-09-17T01:16:30.682 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi033 smithi033 running (10m) 99s ago 22m 63.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de d8d505ca550b 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:mon.smithi100 smithi100 running (9m) 2m ago 20m 48.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 6dbe4cef4690 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.0.0.smithi033.ttskqh smithi033 *:2049 running (2m) 99s ago 14m 58.1M - 5.5 ebb0615149de 1357e68bff5a 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:nfs.foo.1.0.smithi100.mvigkp smithi100 *:2049 running (2m) 2m ago 14m 15.4M - 5.5 ebb0615149de 492568f157ac 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi033 smithi033 *:9100 running (11m) 99s ago 20m 16.5M - 1.5.0 0da6a335fe13 0f299abdcc6b 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.smithi100 smithi100 *:9100 running (11m) 2m ago 20m 16.2M - 1.5.0 0da6a335fe13 3b9692595308 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (8m) 99s ago 19m 421M 1937M 19.1.1-330-gc2ddf972 ebb0615149de fde3bc343bc4 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (7m) 99s ago 18m 343M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 27d9e090360b 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (6m) 99s ago 18m 265M 1937M 19.1.1-330-gc2ddf972 ebb0615149de e9f1082a2cca 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (5m) 99s ago 18m 360M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 0806596cc48f 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi100 running (5m) 2m ago 17m 361M 2703M 19.1.1-330-gc2ddf972 ebb0615149de c317ff1047a4 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi100 running (4m) 2m ago 17m 309M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 6600f8c8da1e 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi100 running (3m) 2m ago 16m 280M 2703M 19.1.1-330-gc2ddf972 ebb0615149de 82283e99afeb 2024-09-17T01:16:30.683 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi100 running (3m) 2m ago 16m 318M 2703M 19.1.1-330-gc2ddf972 ebb0615149de edf5ede3d097 2024-09-17T01:16:30.684 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.smithi033 smithi033 *:9095 running (10m) 99s ago 19m 67.6M - 2.43.0 a07b618ecd1d 5d6c286a09f7 2024-09-17T01:16:31.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:31 smithi033 ceph-mon[65430]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:31.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:31 smithi100 ceph-mon[51701]: from='client.34598 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:31.489 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T01:16:32.477 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:32 smithi033 ceph-mon[65430]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:32.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:32 smithi100 ceph-mon[51701]: pgmap v393: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:33.504 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:16:33.504 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": null, 2024-09-17T01:16:33.504 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": false, 2024-09-17T01:16:33.504 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "", 2024-09-17T01:16:33.504 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-17T01:16:33.504 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": null, 2024-09-17T01:16:33.505 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-17T01:16:33.505 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-17T01:16:33.505 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:16:34.204 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T01:16:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:34 smithi100 ceph-mon[51701]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:34.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:34 smithi100 ceph-mon[51701]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:34.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:34 smithi033 ceph-mon[65430]: pgmap v394: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:34.514 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:34 smithi033 ceph-mon[65430]: from='client.34602 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:36.163 INFO:teuthology.orchestra.run.smithi033.stdout:HEALTH_OK 2024-09-17T01:16:36.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:36 smithi100 ceph-mon[51701]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:36.479 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:36 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3132846880' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:16:36.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:36 smithi033 ceph-mon[65430]: pgmap v395: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:36.614 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:36 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3132846880' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T01:16:36.950 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T01:16:38.272 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:38 smithi033 ceph-mon[65430]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:38.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:38 smithi100 ceph-mon[51701]: pgmap v396: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T01:16:39.142 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:16:39.143 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-17T01:16:39.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:39 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3986520897' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:16:39.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:39 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3986520897' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:16:39.806 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T01:16:40.390 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:40 smithi033 ceph-mon[65430]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:40.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:40 smithi100 ceph-mon[51701]: pgmap v397: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:41.992 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-17T01:16:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:42 smithi033 ceph-mon[65430]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:42 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3504026154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:16:42.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:42 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:42 smithi100 ceph-mon[51701]: pgmap v398: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:42 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3504026154' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:16:42.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:42 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:42.631 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T01:16:44.334 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:44 smithi033 ceph-mon[65430]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:44.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:44 smithi100 ceph-mon[51701]: pgmap v399: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:44.876 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T01:16:45.470 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:45 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/2846051085' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:16:45.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:45 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/2846051085' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T01:16:45.496 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-17T01:16:45.497 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-17T01:16:45.497 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T01:16:46.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:46 smithi100 ceph-mon[51701]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:46.495 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:46 smithi033 ceph-mon[65430]: pgmap v400: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:47.466 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-17T01:16:47.467 INFO:teuthology.orchestra.run.smithi033.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-17T00:55:01.576222Z", "last_refresh": "2024-09-17T01:14:51.412696Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-17T00:54:55.903476Z", "last_refresh": "2024-09-17T01:14:25.156068Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-17T00:54:58.805315Z", "last_refresh": "2024-09-17T01:14:51.412835Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-17T01:01:23.335227Z", "last_refresh": "2024-09-17T01:14:25.156660Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-17T00:54:54.644049Z", "last_refresh": "2024-09-17T01:14:25.156164Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi033:172.21.15.33=smithi033", "smithi100:172.21.15.100=smithi100"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-17T00:55:43.948307Z", "last_refresh": "2024-09-17T01:14:25.156240Z", "running": 2, "size": 2}}, {"events": ["2024-09-17T01:15:56.828910Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-17T01:01:44.274179Z", "last_refresh": "2024-09-17T01:14:25.156729Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-17T00:55:00.041575Z", "last_refresh": "2024-09-17T01:14:25.156311Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", "last_refresh": "2024-09-17T01:14:25.156381Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-17T00:54:57.172843Z", "last_refresh": "2024-09-17T01:14:51.412982Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T01:16:48.248 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-17T01:16:48.248 DEBUG:teuthology.parallel:result is None 2024-09-17T01:16:48.249 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T01:16:48.260 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:16:48.260 DEBUG:teuthology.orchestra.run.smithi033:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-17T01:16:48.289 INFO:teuthology.orchestra.run.smithi033.stderr:+ umount /mnt/foo 2024-09-17T01:16:48.313 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:16:48.326 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:16:48.326 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-17T01:16:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:48 smithi100 ceph-mon[51701]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:48.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:48 smithi100 ceph-mon[51701]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:16:48.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:48 smithi033 ceph-mon[65430]: pgmap v401: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:48.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:48 smithi033 ceph-mon[65430]: from='client.34622 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T01:16:50.456 INFO:teuthology.orchestra.run.smithi033.stdout: "foo" 2024-09-17T01:16:50.456 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:50 smithi033 ceph-mon[65430]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:50.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:50 smithi100 ceph-mon[51701]: pgmap v402: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T01:16:51.119 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-17T01:16:51.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:51 smithi033 ceph-mon[65430]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:51.481 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:51 smithi100 ceph-mon[51701]: from='client.34626 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:52.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:52 smithi100 ceph-mon[51701]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:52.503 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:52 smithi033 ceph-mon[65430]: pgmap v403: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout:[ 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: { 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "access_type": "RW", 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "clients": [], 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "cluster_id": "foo", 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "export_id": 1, 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "fsal": { 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "fs_name": "foofs", 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "name": "CEPH", 2024-09-17T01:16:53.277 INFO:teuthology.orchestra.run.smithi033.stdout: "user_id": "nfs.foo.1" 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "path": "/", 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "protocols": [ 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: 4 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "pseudo": "/fake", 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "security_label": true, 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "squash": "none", 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "transports": [ 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: "TCP" 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: ] 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-17T01:16:53.278 INFO:teuthology.orchestra.run.smithi033.stdout:] 2024-09-17T01:16:53.933 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-17T01:16:54.512 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:54 smithi033 ceph-mon[65430]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:54.512 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:54 smithi033 ceph-mon[65430]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:54.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:54 smithi100 ceph-mon[51701]: pgmap v404: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:54.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:54 smithi100 ceph-mon[51701]: from='client.34630 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T01:16:55.761 INFO:teuthology.orchestra.run.smithi033.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-17T01:16:55.761 INFO:teuthology.orchestra.run.smithi033.stdout:foo grace 2024-09-17T01:16:55.762 INFO:teuthology.orchestra.run.smithi033.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-17T01:16:55.762 INFO:teuthology.orchestra.run.smithi033.stdout:foo conf-nfs.foo 2024-09-17T01:16:55.762 INFO:teuthology.orchestra.run.smithi033.stdout:foo export-1 2024-09-17T01:16:56.449 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:56 smithi033 ceph-mon[65430]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:56.449 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:56 smithi033 ceph-mon[65430]: mgrmap e43: smithi033.illmog(active, since 10m), standbys: smithi100.qrwkie 2024-09-17T01:16:56.477 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T01:16:56.489 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:16:56.489 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 47eab686-748f-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-17T01:16:56.490 DEBUG:teuthology.orchestra.run.smithi033:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-17T01:16:56.490 DEBUG:teuthology.orchestra.run.smithi033:> ' 2024-09-17T01:16:56.529 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:56 smithi100 ceph-mon[51701]: pgmap v405: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T01:16:56.530 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:56 smithi100 ceph-mon[51701]: mgrmap e43: smithi033.illmog(active, since 10m), standbys: smithi100.qrwkie 2024-09-17T01:16:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:16:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:57.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:57 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:57.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T01:16:57.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T01:16:57.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:57.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:57 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:58.318 INFO:teuthology.orchestra.run.smithi033.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-17T01:16:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:58 smithi033 ceph-mon[65430]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T01:16:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:16:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:16:58.363 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:58.364 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:58 smithi033 ceph-mon[65430]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:58.561 INFO:teuthology.orchestra.run.smithi033.stderr:+ [[ 7 -gt 2 ]] 2024-09-17T01:16:58.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:58 smithi100 ceph-mon[51701]: pgmap v406: 97 pgs: 97 active+clean; 232 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T01:16:58.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T01:16:58.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T01:16:58.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:58.728 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:58 smithi100 ceph-mon[51701]: from='mgr.34108 172.21.15.33:0/2812109065' entity='mgr.smithi033.illmog' 2024-09-17T01:16:59.323 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:59 smithi033 ceph-mon[65430]: from='client.? 172.21.15.33:0/3945332928' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-17T01:16:59.353 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T01:16:59.364 INFO:tasks.cephadm:Teardown begin 2024-09-17T01:16:59.364 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:16:59.395 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:16:59.429 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T01:16:59.429 DEBUG:teuthology.orchestra.run.smithi033:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T01:16:59.453 DEBUG:teuthology.orchestra.run.smithi100:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T01:16:59.488 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T01:16:59.489 INFO:tasks.cephadm.mon.smithi033:Stopping mon.smithi033... 2024-09-17T01:16:59.489 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033 2024-09-17T01:16:59.613 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:59 smithi033 systemd[1]: Stopping Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:16:59.727 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:16:59 smithi100 ceph-mon[51701]: from='client.? 172.21.15.33:0/3945332928' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-17T01:17:00.061 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[65426]: 2024-09-17T01:16:59.787+0000 7fe2c4ede640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi033 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:00.061 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:59 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033[65426]: 2024-09-17T01:16:59.787+0000 7fe2c4ede640 -1 mon.smithi033@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T01:17:00.061 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:59 smithi033 podman[98399]: 2024-09-17 01:16:59.951459285 +0000 UTC m=+0.259769853 container stop d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T01:17:00.062 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:16:59 smithi033 podman[98399]: 2024-09-17 01:16:59.977157341 +0000 UTC m=+0.285467909 container died d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T01:17:00.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:17:00 smithi033 podman[98399]: 2024-09-17 01:17:00.387798772 +0000 UTC m=+0.696109340 container cleanup d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:17:00.587 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:17:00 smithi033 bash[98399]: ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033 2024-09-17T01:17:00.821 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service' 2024-09-17T01:17:00.844 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:17:00 smithi033 podman[98410]: 2024-09-17 01:17:00.589352494 +0000 UTC m=+0.631882779 container remove d8d505ca550b9d415f8bab79b08fb6ec138e35a21bdaf910fa616aa8edfcf8b1 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi033, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T01:17:00.844 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:17:00 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service: Deactivated successfully. 2024-09-17T01:17:00.844 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:17:00 smithi033 systemd[1]: Stopped Ceph mon.smithi033 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:17:00.844 INFO:journalctl@ceph.mon.smithi033.smithi033.stdout:Sep 17 01:17:00 smithi033 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi033.service: Consumed 13.633s CPU time. 2024-09-17T01:17:00.885 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:00.886 INFO:tasks.cephadm.mon.smithi033:Stopped mon.smithi033 2024-09-17T01:17:00.886 INFO:tasks.cephadm.mon.smithi100:Stopping mon.smithi100... 2024-09-17T01:17:00.886 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi100 2024-09-17T01:17:01.212 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:17:00 smithi100 systemd[1]: Stopping Ceph mon.smithi100 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:01.477 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:17:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100[51697]: 2024-09-17T01:17:01.213+0000 7f4876122640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi100 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:01.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:17:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100[51697]: 2024-09-17T01:17:01.213+0000 7f4876122640 -1 mon.smithi100@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T01:17:01.478 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:17:01 smithi100 podman[73741]: 2024-09-17 01:17:01.275087409 +0000 UTC m=+0.188460091 container died 6dbe4cef4690a7f56075539214087ca268b61596a9034260391fd925731ac5f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:17:01 smithi100 podman[73741]: 2024-09-17 01:17:01.877745468 +0000 UTC m=+0.791118147 container cleanup 6dbe4cef4690a7f56075539214087ca268b61596a9034260391fd925731ac5f7 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T01:17:01.977 INFO:journalctl@ceph.mon.smithi100.smithi100.stdout:Sep 17 01:17:01 smithi100 bash[73741]: ceph-47eab686-748f-11ef-bceb-c7b262605968-mon-smithi100 2024-09-17T01:17:02.306 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@mon.smithi100.service' 2024-09-17T01:17:02.355 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:02.355 INFO:tasks.cephadm.mon.smithi100:Stopped mon.smithi100 2024-09-17T01:17:02.355 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T01:17:02.355 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.0 2024-09-17T01:17:02.657 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:02 smithi033 systemd[1]: Stopping Ceph osd.0 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:03.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:02 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:17:02.659+0000 7fb3a8526640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:03.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:02 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:17:02.659+0000 7fb3a8526640 -1 osd.0 160 *** Got signal Terminated *** 2024-09-17T01:17:03.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:02 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0[71517]: 2024-09-17T01:17:02.659+0000 7fb3a8526640 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:08.491 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:08 smithi033 podman[98563]: 2024-09-17 01:17:08.232847942 +0000 UTC m=+5.669272686 container died fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T01:17:08.761 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:08 smithi033 podman[98563]: 2024-09-17 01:17:08.567376236 +0000 UTC m=+6.003800971 container cleanup fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97) 2024-09-17T01:17:08.761 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:08 smithi033 bash[98563]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0 2024-09-17T01:17:09.060 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:08 smithi033 podman[98579]: 2024-09-17 01:17:08.760938704 +0000 UTC m=+0.529891410 container remove fde3bc343bc41e6be1825c2d5536a17b3af5025c1fe8bbc4465856c4e292a057 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-17T01:17:09.342 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:09 smithi033 podman[98667]: 2024-09-17 01:17:09.101929344 +0000 UTC m=+0.016632660 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:09.342 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:09 smithi033 podman[98667]: 2024-09-17 01:17:09.220317348 +0000 UTC m=+0.135020652 container create 13473c1707ea5aa576bd8fae37d16e96ac6fce2215d0b51bbc2114a0225aa8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T01:17:09.613 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:09 smithi033 podman[98667]: 2024-09-17 01:17:09.489955729 +0000 UTC m=+0.404659037 container init 13473c1707ea5aa576bd8fae37d16e96ac6fce2215d0b51bbc2114a0225aa8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T01:17:09.613 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:09 smithi033 podman[98667]: 2024-09-17 01:17:09.493525001 +0000 UTC m=+0.408228305 container start 13473c1707ea5aa576bd8fae37d16e96ac6fce2215d0b51bbc2114a0225aa8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T01:17:09.613 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:09 smithi033 podman[98667]: 2024-09-17 01:17:09.5649645 +0000 UTC m=+0.479667818 container attach 13473c1707ea5aa576bd8fae37d16e96ac6fce2215d0b51bbc2114a0225aa8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:09.993 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:09 smithi033 podman[98667]: 2024-09-17 01:17:09.691991219 +0000 UTC m=+0.606694541 container died 13473c1707ea5aa576bd8fae37d16e96ac6fce2215d0b51bbc2114a0225aa8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T01:17:10.255 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 17 01:17:10 smithi033 podman[98690]: 2024-09-17 01:17:10.236012743 +0000 UTC m=+0.533987020 container remove 13473c1707ea5aa576bd8fae37d16e96ac6fce2215d0b51bbc2114a0225aa8f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T01:17:10.261 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T01:17:10.302 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:10.302 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T01:17:10.302 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T01:17:10.303 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.1 2024-09-17T01:17:10.613 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:10 smithi033 systemd[1]: Stopping Ceph osd.1 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:11.113 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:10 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:17:10.672+0000 7f6ff37be640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:11.113 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:10 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:17:10.672+0000 7f6ff37be640 -1 osd.1 160 *** Got signal Terminated *** 2024-09-17T01:17:11.113 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:10 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1[74157]: 2024-09-17T01:17:10.672+0000 7f6ff37be640 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:16.321 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:16 smithi033 podman[98769]: 2024-09-17 01:17:16.066969625 +0000 UTC m=+5.516683851 container died 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:17:16.587 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:16 smithi033 podman[98769]: 2024-09-17 01:17:16.397200296 +0000 UTC m=+5.846914506 container cleanup 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:17:16.588 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:16 smithi033 bash[98769]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1 2024-09-17T01:17:16.863 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:16 smithi033 podman[98787]: 2024-09-17 01:17:16.590239076 +0000 UTC m=+0.523926016 container remove 27d9e090360b147122da57c1e7b39c2f0ed61bee81e588ff208cebb00bb3fbcd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:17.309 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:17 smithi033 podman[98875]: 2024-09-17 01:17:16.926985395 +0000 UTC m=+0.017141879 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:17.309 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:17 smithi033 podman[98875]: 2024-09-17 01:17:17.024579394 +0000 UTC m=+0.114735926 container create 80ca12df9b6b8196783eb52cb02c5970f6277ab77ac54658438b45d738b2bdbb (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97) 2024-09-17T01:17:17.309 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:17 smithi033 podman[98875]: 2024-09-17 01:17:17.311276927 +0000 UTC m=+0.401433420 container init 80ca12df9b6b8196783eb52cb02c5970f6277ab77ac54658438b45d738b2bdbb (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2) 2024-09-17T01:17:17.613 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:17 smithi033 podman[98875]: 2024-09-17 01:17:17.315126024 +0000 UTC m=+0.405282513 container start 80ca12df9b6b8196783eb52cb02c5970f6277ab77ac54658438b45d738b2bdbb (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T01:17:17.614 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:17 smithi033 podman[98875]: 2024-09-17 01:17:17.386243814 +0000 UTC m=+0.476400298 container attach 80ca12df9b6b8196783eb52cb02c5970f6277ab77ac54658438b45d738b2bdbb (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-17T01:17:17.614 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 17 01:17:17 smithi033 podman[98875]: 2024-09-17 01:17:17.507653608 +0000 UTC m=+0.597810101 container died 80ca12df9b6b8196783eb52cb02c5970f6277ab77ac54658438b45d738b2bdbb (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:17:18.086 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T01:17:18.129 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:18.129 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T01:17:18.129 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T01:17:18.130 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.2 2024-09-17T01:17:18.364 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:18 smithi033 systemd[1]: Stopping Ceph osd.2 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:18.863 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:18 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:17:18.484+0000 7f4c2c7f0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:18.863 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:18 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:17:18.484+0000 7f4c2c7f0640 -1 osd.2 160 *** Got signal Terminated *** 2024-09-17T01:17:18.863 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:18 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2[76883]: 2024-09-17T01:17:18.484+0000 7f4c2c7f0640 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:24.308 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:24 smithi033 podman[98976]: 2024-09-17 01:17:24.034133116 +0000 UTC m=+5.665787286 container died e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True) 2024-09-17T01:17:24.613 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:24 smithi033 podman[98976]: 2024-09-17 01:17:24.384919629 +0000 UTC m=+6.016573800 container cleanup e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:24.613 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:24 smithi033 bash[98976]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2 2024-09-17T01:17:24.967 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:24 smithi033 podman[99023]: 2024-09-17 01:17:24.64434047 +0000 UTC m=+0.610610174 container remove e9f1082a2ccaf539f2dc2c2a024a582979167ee6a47c917cb56dfd279e865d54 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:17:25.261 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:24 smithi033 podman[99111]: 2024-09-17 01:17:24.970231545 +0000 UTC m=+0.113624149 container create f39b864d17b443e67f17fb3f19d326fd0592457476a7d463dff2e00a16505a3e (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T01:17:25.262 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:24 smithi033 podman[99111]: 2024-09-17 01:17:24.873315094 +0000 UTC m=+0.016707709 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:25.262 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:25 smithi033 podman[99111]: 2024-09-17 01:17:25.189760389 +0000 UTC m=+0.333152998 container init f39b864d17b443e67f17fb3f19d326fd0592457476a7d463dff2e00a16505a3e (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T01:17:25.262 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:25 smithi033 podman[99111]: 2024-09-17 01:17:25.193496421 +0000 UTC m=+0.336889035 container start f39b864d17b443e67f17fb3f19d326fd0592457476a7d463dff2e00a16505a3e (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T01:17:25.262 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:25 smithi033 podman[99111]: 2024-09-17 01:17:25.264312347 +0000 UTC m=+0.407704969 container attach f39b864d17b443e67f17fb3f19d326fd0592457476a7d463dff2e00a16505a3e (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T01:17:25.613 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 17 01:17:25 smithi033 podman[99111]: 2024-09-17 01:17:25.390060811 +0000 UTC m=+0.533453425 container died f39b864d17b443e67f17fb3f19d326fd0592457476a7d463dff2e00a16505a3e (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T01:17:25.985 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T01:17:26.026 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:26.027 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T01:17:26.027 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T01:17:26.027 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.3 2024-09-17T01:17:26.353 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:26 smithi033 systemd[1]: Stopping Ceph osd.3 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:26.613 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:26 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:17:26.354+0000 7fed678f1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:26.613 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:26 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:17:26.354+0000 7fed678f1640 -1 osd.3 160 *** Got signal Terminated *** 2024-09-17T01:17:26.613 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:26 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:17:26.354+0000 7fed678f1640 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:29.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:29 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:29.678+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:30.113 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:29 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:17:29.846+0000 7fed63f07640 -1 osd.3 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.037521+0000 front 2024-09-17T01:17:06.037545+0000 (oldest deadline 2024-09-17T01:17:29.537605+0000) 2024-09-17T01:17:30.444 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:30 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:30.048+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:30.977 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:30 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:30.698+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:31.113 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:30 smithi033 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3[79484]: 2024-09-17T01:17:30.816+0000 7fed63f07640 -1 osd.3 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.037521+0000 front 2024-09-17T01:17:06.037545+0000 (oldest deadline 2024-09-17T01:17:29.537605+0000) 2024-09-17T01:17:31.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:31 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:31.145+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:31.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:31 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:31.097+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:32.055 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:31 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:31.731+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:32.056 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:31 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:31.923+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:32.136 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:31 smithi033 podman[99212]: 2024-09-17 01:17:31.875268083 +0000 UTC m=+5.608730334 container died 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-17T01:17:32.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:32 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:32.109+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:32.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:32 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:32.057+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:32.613 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:32 smithi033 podman[99212]: 2024-09-17 01:17:32.21260232 +0000 UTC m=+5.946064562 container cleanup 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-17T01:17:32.613 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:32 smithi033 bash[99212]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3 2024-09-17T01:17:32.613 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:32 smithi033 podman[99227]: 2024-09-17 01:17:32.380679735 +0000 UTC m=+0.505992726 container remove 0806596cc48f12ec15c17836916753c8aa407ed732b902686be4fe61acbfa988 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:17:33.037 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:32 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:32.777+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:33.038 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:32 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:32.923+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:33.073 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:32 smithi033 podman[99320]: 2024-09-17 01:17:32.701704653 +0000 UTC m=+0.017156001 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:33.073 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:32 smithi033 podman[99320]: 2024-09-17 01:17:32.814911153 +0000 UTC m=+0.130362524 container create ca9e1a06935123f4589e8d09251333d3e0edd0890884f0748c61b7530b3a8937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T01:17:33.073 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:33 smithi033 podman[99320]: 2024-09-17 01:17:33.075960115 +0000 UTC m=+0.391411463 container init ca9e1a06935123f4589e8d09251333d3e0edd0890884f0748c61b7530b3a8937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True) 2024-09-17T01:17:33.364 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:33 smithi033 podman[99320]: 2024-09-17 01:17:33.079589152 +0000 UTC m=+0.395040480 container start ca9e1a06935123f4589e8d09251333d3e0edd0890884f0748c61b7530b3a8937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T01:17:33.364 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:33 smithi033 podman[99320]: 2024-09-17 01:17:33.159247201 +0000 UTC m=+0.474698562 container attach ca9e1a06935123f4589e8d09251333d3e0edd0890884f0748c61b7530b3a8937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:17:33.364 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:33 smithi033 podman[99320]: 2024-09-17 01:17:33.272061047 +0000 UTC m=+0.587512383 container died ca9e1a06935123f4589e8d09251333d3e0edd0890884f0748c61b7530b3a8937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS) 2024-09-17T01:17:33.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:33.151+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:33.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:33.040+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:33.807 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 17 01:17:33 smithi033 podman[99343]: 2024-09-17 01:17:33.796361035 +0000 UTC m=+0.517527949 container remove ca9e1a06935123f4589e8d09251333d3e0edd0890884f0748c61b7530b3a8937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T01:17:33.813 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T01:17:33.855 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:33.855 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T01:17:33.855 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T01:17:33.855 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4 2024-09-17T01:17:34.055 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:33.766+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:34.055 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:33 smithi100 systemd[1]: Stopping Ceph osd.4 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:34.055 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:33 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:33.892+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:34.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:34.133+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:34.477 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:34.057+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:34.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:34.165+0000 7f9753e70640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:34.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:34.165+0000 7f9753e70640 -1 osd.4 160 *** Got signal Terminated *** 2024-09-17T01:17:34.478 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:34.165+0000 7f9753e70640 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:35.168 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:35 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:35.031+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:35.168 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:34.810+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:35.168 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:34 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:34.891+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:35.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:35 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:35.169+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:36.079 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:35 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:35.793+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:36.080 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:35 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:35.872+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:36.477 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:36 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:36.082+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:36.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:36 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:36.176+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:37.212 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:36 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:36.834+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:37.212 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:37 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:37.047+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:37.212 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:36 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:36.910+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:37.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:37 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:37.214+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:38.218 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:37 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:37.790+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:38.218 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:38 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:37.999+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:38.218 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:37 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:37.870+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:38.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:38 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:38.220+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:38.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:38 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:38.220+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:39.183 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:38 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4[56609]: 2024-09-17T01:17:38.787+0000 7f974c47e640 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:04.542338+0000 front 2024-09-17T01:17:04.542351+0000 (oldest deadline 2024-09-17T01:17:29.242108+0000) 2024-09-17T01:17:39.183 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:39 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:39.017+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:39.184 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:38 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:38.876+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:39.184 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:38 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:38.876+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:39.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:39 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:39.185+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:39.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:39 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:39.185+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:39.872 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:39 smithi100 podman[73907]: 2024-09-17 01:17:39.586126066 +0000 UTC m=+5.528693780 container died c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T01:17:40.185 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:40 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:40.046+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:40.185 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:40 smithi100 podman[73907]: 2024-09-17 01:17:40.049355263 +0000 UTC m=+5.991922965 container cleanup c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T01:17:40.185 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:40 smithi100 bash[73907]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4 2024-09-17T01:17:40.186 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:39 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:39.889+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:40.186 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:39 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:39.889+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:40.477 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:40 smithi100 podman[73924]: 2024-09-17 01:17:40.418612963 +0000 UTC m=+0.835694787 container remove c317ff1047a4a3d312cde1bdda9fe90d8caeac6cbaaee4bb43da641aee16f961 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0) 2024-09-17T01:17:40.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:40 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:40.187+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:40.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:40 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:40.187+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:40.966 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:40 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:40.860+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:40.966 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:40 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:40.860+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:40.966 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:40 smithi100 podman[74014]: 2024-09-17 01:17:40.720245256 +0000 UTC m=+0.015613302 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:40.966 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:40 smithi100 podman[74014]: 2024-09-17 01:17:40.844521181 +0000 UTC m=+0.139889228 container create e50c3a7f4485179f4325fabae7be5e0b34c40cfffb437398d2607698fed5a5e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T01:17:41.227 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:41.156+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:41.227 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:41.156+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:41.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:41.042+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:41.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:41.042+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:41.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 podman[74014]: 2024-09-17 01:17:41.09288081 +0000 UTC m=+0.388248858 container init e50c3a7f4485179f4325fabae7be5e0b34c40cfffb437398d2607698fed5a5e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:41.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 podman[74014]: 2024-09-17 01:17:41.096146022 +0000 UTC m=+0.391514077 container start e50c3a7f4485179f4325fabae7be5e0b34c40cfffb437398d2607698fed5a5e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T01:17:41.228 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 podman[74014]: 2024-09-17 01:17:41.178159647 +0000 UTC m=+0.473527711 container attach e50c3a7f4485179f4325fabae7be5e0b34c40cfffb437398d2607698fed5a5e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97) 2024-09-17T01:17:41.683 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 podman[74014]: 2024-09-17 01:17:41.30089253 +0000 UTC m=+0.596260580 container died e50c3a7f4485179f4325fabae7be5e0b34c40cfffb437398d2607698fed5a5e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T01:17:41.928 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T01:17:41.952 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:41.825+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:41.952 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:41.825+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:41.952 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 podman[74071]: 2024-09-17 01:17:41.913968651 +0000 UTC m=+0.606998877 container remove e50c3a7f4485179f4325fabae7be5e0b34c40cfffb437398d2607698fed5a5e2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97) 2024-09-17T01:17:41.952 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T01:17:41.953 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 systemd[1]: Stopped Ceph osd.4 for 47eab686-748f-11ef-bceb-c7b262605968. 2024-09-17T01:17:41.953 INFO:journalctl@ceph.osd.4.smithi100.stdout:Sep 17 01:17:41 smithi100 systemd[1]: ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.4.service: Consumed 1min 8.491s CPU time. 2024-09-17T01:17:41.967 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:41.967 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T01:17:41.968 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T01:17:41.968 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.5 2024-09-17T01:17:42.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:41.993+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:42.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:41 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:41.993+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:42.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:42 smithi100 systemd[1]: Stopping Ceph osd.5 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:42.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:42.139+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:42.228 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:42.139+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:42.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:42.325+0000 7f47d40c8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:42.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:42.325+0000 7f47d40c8640 -1 osd.5 160 *** Got signal Terminated *** 2024-09-17T01:17:42.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:42.325+0000 7f47d40c8640 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:43.161 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:42.798+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:43.161 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:42.798+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:43.161 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:42.948+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:43.161 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:42 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:42.948+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:43.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:43 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:43.163+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:43.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:43 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:43.163+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:44.171 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:43 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:43.754+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:44.171 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:43 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:43.754+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:44.171 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:43 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:43.927+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:44.171 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:43 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:43.927+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:44.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:44.173+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:44.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:44.173+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:45.180 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:44.905+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:45.180 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:44.905+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:45.180 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:44.764+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:45.180 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:44.764+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:45.181 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:44 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:44.764+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:45.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:45.182+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:45.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:45.182+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:45.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:45.182+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:18.035666+0000 front 2024-09-17T01:17:18.035709+0000 (oldest deadline 2024-09-17T01:17:44.435894+0000) 2024-09-17T01:17:46.146 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:45.886+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:46.146 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:45.886+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:46.147 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:45.753+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:46.147 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:45.753+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:46.147 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:45 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:45.753+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:46.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:46.149+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:46.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:46.149+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:46.477 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:46.149+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:18.035666+0000 front 2024-09-17T01:17:18.035709+0000 (oldest deadline 2024-09-17T01:17:44.435894+0000) 2024-09-17T01:17:47.175 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:46.840+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:47.175 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:46.840+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:47.175 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:46.769+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:47.175 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:46.769+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:47.176 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:46 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:46.769+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:47.480 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:47.177+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:05.134381+0000 front 2024-09-17T01:17:05.134405+0000 (oldest deadline 2024-09-17T01:17:30.434201+0000) 2024-09-17T01:17:47.481 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:47.177+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.135252+0000 front 2024-09-17T01:17:12.135327+0000 (oldest deadline 2024-09-17T01:17:38.035110+0000) 2024-09-17T01:17:47.481 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5[58846]: 2024-09-17T01:17:47.177+0000 7f47d06de640 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:18.035666+0000 front 2024-09-17T01:17:18.035709+0000 (oldest deadline 2024-09-17T01:17:44.435894+0000) 2024-09-17T01:17:48.007 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:47 smithi100 podman[74161]: 2024-09-17 01:17:47.736598187 +0000 UTC m=+5.542736430 container died 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:17:48.007 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:47.811+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:48.008 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:47.811+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:48.008 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:47.811+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:48.008 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:47.798+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:48.008 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:47.798+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:48.008 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:47 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:47.798+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:48.474 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:48 smithi100 podman[74161]: 2024-09-17 01:17:48.217638528 +0000 UTC m=+6.023776835 container cleanup 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T01:17:48.474 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:48 smithi100 bash[74161]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5 2024-09-17T01:17:48.727 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:48 smithi100 podman[74178]: 2024-09-17 01:17:48.47729337 +0000 UTC m=+0.738639956 container remove 6600f8c8da1e6f3e683517eb0252b8304bf1b4b493f5a199283943daf2497029 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-17T01:17:49.061 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:48 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:48.806+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:49.061 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:48 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:48.806+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:49.061 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:48 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:48.806+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:49.061 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:48 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:48.791+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:49.061 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:48 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:48.791+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:49.062 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:48 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:48.791+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:49.062 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:48 smithi100 podman[74268]: 2024-09-17 01:17:48.737857489 +0000 UTC m=+0.015791993 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:49.062 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:48 smithi100 podman[74268]: 2024-09-17 01:17:48.844877918 +0000 UTC m=+0.122812409 container create af89fef9ad8523b20b4711b55cdf4332e697e9f0c7d4c5195f5aa6e1e7e36b5f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-17T01:17:49.062 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:49 smithi100 podman[74268]: 2024-09-17 01:17:49.063895245 +0000 UTC m=+0.341829737 container init af89fef9ad8523b20b4711b55cdf4332e697e9f0c7d4c5195f5aa6e1e7e36b5f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-17T01:17:49.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:49 smithi100 podman[74268]: 2024-09-17 01:17:49.067435355 +0000 UTC m=+0.345369847 container start af89fef9ad8523b20b4711b55cdf4332e697e9f0c7d4c5195f5aa6e1e7e36b5f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:49.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:49 smithi100 podman[74268]: 2024-09-17 01:17:49.155507316 +0000 UTC m=+0.433441841 container attach af89fef9ad8523b20b4711b55cdf4332e697e9f0c7d4c5195f5aa6e1e7e36b5f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T01:17:49.478 INFO:journalctl@ceph.osd.5.smithi100.stdout:Sep 17 01:17:49 smithi100 podman[74268]: 2024-09-17 01:17:49.260998611 +0000 UTC m=+0.538933103 container died af89fef9ad8523b20b4711b55cdf4332e697e9f0c7d4c5195f5aa6e1e7e36b5f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T01:17:49.900 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:49 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:49.794+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:49.900 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:49 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:49.794+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:49.901 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:49 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:49.795+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:49.901 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:49 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:49.818+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:49.901 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:49 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:49.818+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:49.901 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:49 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:49.818+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:49.925 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T01:17:49.967 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:49.967 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T01:17:49.967 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T01:17:49.968 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.6 2024-09-17T01:17:50.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 systemd[1]: Stopping Ceph osd.6 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:50.727 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:50.322+0000 7f738390d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:50.727 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:50.322+0000 7f738390d640 -1 osd.6 160 *** Got signal Terminated *** 2024-09-17T01:17:50.728 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:50.322+0000 7f738390d640 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:51.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:50.817+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:51.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:50.817+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:51.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:50.817+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:51.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:50.782+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:51.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:50.782+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:51.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:50 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:50.782+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:52.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:51 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:51.834+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:52.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:51 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:51.834+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:52.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:51 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:51.834+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:52.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:51 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:51.803+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:52.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:51 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:51.803+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:52.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:51 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:51.803+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:53.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:52.790+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:52.790+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:52.790+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:52.790+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:30.272921+0000 front 2024-09-17T01:17:30.272883+0000 (oldest deadline 2024-09-17T01:17:52.572742+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:52.772+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:52.772+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:52.772+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:53.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:52 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:52.772+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:54.227 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:53.840+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:53.840+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:53.840+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:53.840+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:30.272921+0000 front 2024-09-17T01:17:30.272883+0000 (oldest deadline 2024-09-17T01:17:52.572742+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:53.805+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:53.805+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:53.805+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:54.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:53 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:53.805+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:55.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:54.845+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:03.270441+0000 front 2024-09-17T01:17:03.270393+0000 (oldest deadline 2024-09-17T01:17:29.170270+0000) 2024-09-17T01:17:55.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:54.845+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:15.071223+0000 front 2024-09-17T01:17:15.071323+0000 (oldest deadline 2024-09-17T01:17:40.971144+0000) 2024-09-17T01:17:55.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:54.845+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:20.971789+0000 front 2024-09-17T01:17:20.971761+0000 (oldest deadline 2024-09-17T01:17:46.871574+0000) 2024-09-17T01:17:55.228 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6[61097]: 2024-09-17T01:17:54.845+0000 7f737bf1b640 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:30.272921+0000 front 2024-09-17T01:17:30.272883+0000 (oldest deadline 2024-09-17T01:17:52.572742+0000) 2024-09-17T01:17:55.229 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:54.789+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:55.229 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:54.789+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:55.229 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:54.789+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:55.229 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:54 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:54.789+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:56.187 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:55 smithi100 podman[74371]: 2024-09-17 01:17:55.912203729 +0000 UTC m=+5.717332121 container died 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T01:17:56.187 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:55 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:55.750+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:56.187 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:55 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:55.750+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:56.188 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:55 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:55.750+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:56.188 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:55 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:55.750+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:56.477 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:56 smithi100 podman[74371]: 2024-09-17 01:17:56.272378396 +0000 UTC m=+6.077506823 container cleanup 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T01:17:56.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:56 smithi100 bash[74371]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6 2024-09-17T01:17:56.887 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:56 smithi100 podman[74386]: 2024-09-17 01:17:56.565432664 +0000 UTC m=+0.647010613 container remove 82283e99afeb676d2d23cf16a76245b76093ae0b84793ede12d1b3788f5d80d9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T01:17:56.888 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:56.702+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:56.888 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:56.702+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:56.888 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:56.702+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:56.888 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:56 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:56.702+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:57.149 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:56 smithi100 podman[74476]: 2024-09-17 01:17:56.793407897 +0000 UTC m=+0.015734813 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:17:57.149 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:56 smithi100 podman[74476]: 2024-09-17 01:17:56.933372135 +0000 UTC m=+0.155699090 container create adcc23a632f3e783f2ba9bbfcaac87e0730c77809ee3aa3ab80b16e1034425bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-17T01:17:57.149 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:57 smithi100 podman[74476]: 2024-09-17 01:17:57.15230929 +0000 UTC m=+0.374636211 container init adcc23a632f3e783f2ba9bbfcaac87e0730c77809ee3aa3ab80b16e1034425bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T01:17:57.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:57 smithi100 podman[74476]: 2024-09-17 01:17:57.15556422 +0000 UTC m=+0.377891130 container start adcc23a632f3e783f2ba9bbfcaac87e0730c77809ee3aa3ab80b16e1034425bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T01:17:57.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:57 smithi100 podman[74476]: 2024-09-17 01:17:57.244003246 +0000 UTC m=+0.466330160 container attach adcc23a632f3e783f2ba9bbfcaac87e0730c77809ee3aa3ab80b16e1034425bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T01:17:57.478 INFO:journalctl@ceph.osd.6.smithi100.stdout:Sep 17 01:17:57 smithi100 podman[74476]: 2024-09-17 01:17:57.345406273 +0000 UTC m=+0.567733192 container died adcc23a632f3e783f2ba9bbfcaac87e0730c77809ee3aa3ab80b16e1034425bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0) 2024-09-17T01:17:57.920 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:57 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:57.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:57.920 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:57 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:57.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:57.920 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:57 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:57.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:57.920 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:57 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:57.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:57.943 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T01:17:57.980 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:17:57.981 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T01:17:57.981 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T01:17:57.981 DEBUG:teuthology.orchestra.run.smithi100:> sudo systemctl stop ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.7 2024-09-17T01:17:58.227 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 systemd[1]: Stopping Ceph osd.7 for 47eab686-748f-11ef-bceb-c7b262605968... 2024-09-17T01:17:58.721 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.324+0000 7f02df2c6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T01:17:58.721 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.324+0000 7f02df2c6640 -1 osd.7 160 *** Got signal Terminated *** 2024-09-17T01:17:58.721 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.324+0000 7f02df2c6640 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T01:17:58.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:58.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:58.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:58.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:58 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:58.722+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:17:59.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:59 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:59.701+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:17:59.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:59 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:59.701+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:17:59.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:59 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:59.701+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:17:59.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:17:59 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:17:59.701+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:18:00.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:00 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:00.698+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:18:00.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:00 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:00.698+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:18:00.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:00 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:00.698+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:18:00.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:00 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:00.698+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:18:00.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:00 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:00.698+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.100:6804 osd.4 since back 2024-09-17T01:17:36.066011+0000 front 2024-09-17T01:17:36.066075+0000 (oldest deadline 2024-09-17T01:18:00.165983+0000) 2024-09-17T01:18:01.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:01.721+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:18:01.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:01.721+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:18:01.977 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:01.721+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:18:01.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:01.721+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:18:01.978 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:01 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:01.721+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.100:6804 osd.4 since back 2024-09-17T01:17:36.066011+0000 front 2024-09-17T01:17:36.066075+0000 (oldest deadline 2024-09-17T01:18:00.165983+0000) 2024-09-17T01:18:03.227 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:02 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:02.744+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-17T01:17:06.863118+0000 front 2024-09-17T01:17:06.863219+0000 (oldest deadline 2024-09-17T01:17:31.563011+0000) 2024-09-17T01:18:03.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:02 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:02.744+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-17T01:17:12.063780+0000 front 2024-09-17T01:17:12.063752+0000 (oldest deadline 2024-09-17T01:17:37.963620+0000) 2024-09-17T01:18:03.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:02 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:02.744+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-17T01:17:17.964127+0000 front 2024-09-17T01:17:17.964092+0000 (oldest deadline 2024-09-17T01:17:44.364331+0000) 2024-09-17T01:18:03.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:02 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:02.744+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-17T01:17:28.465335+0000 front 2024-09-17T01:17:28.465286+0000 (oldest deadline 2024-09-17T01:17:51.965168+0000) 2024-09-17T01:18:03.228 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:02 smithi100 ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7[63361]: 2024-09-17T01:18:02.744+0000 7f02db8dc640 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.100:6804 osd.4 since back 2024-09-17T01:17:36.066011+0000 front 2024-09-17T01:17:36.066075+0000 (oldest deadline 2024-09-17T01:18:00.165983+0000) 2024-09-17T01:18:04.036 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:03 smithi100 podman[74579]: 2024-09-17 01:18:03.730601476 +0000 UTC m=+5.521770416 container died edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-17T01:18:04.457 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:04 smithi100 podman[74579]: 2024-09-17 01:18:04.209252631 +0000 UTC m=+6.000421495 container cleanup edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-17T01:18:04.458 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:04 smithi100 bash[74579]: ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7 2024-09-17T01:18:04.728 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:04 smithi100 podman[74642]: 2024-09-17 01:18:04.460424947 +0000 UTC m=+0.720522505 container remove edf5ede3d09725934dbd369f8a66adc5cb816e0dae103c68598f4bf7122ac92d (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T01:18:05.096 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:04 smithi100 podman[74730]: 2024-09-17 01:18:04.675148711 +0000 UTC m=+0.015940568 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T01:18:05.096 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:04 smithi100 podman[74730]: 2024-09-17 01:18:04.794730886 +0000 UTC m=+0.135522739 container create 122a96d4e4f399c20f58e25e5e7f68499dd6284f860bc818d7a7e142d7ed1f4f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T01:18:05.097 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:05 smithi100 podman[74730]: 2024-09-17 01:18:05.014108821 +0000 UTC m=+0.354900696 container init 122a96d4e4f399c20f58e25e5e7f68499dd6284f860bc818d7a7e142d7ed1f4f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T01:18:05.097 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:05 smithi100 podman[74730]: 2024-09-17 01:18:05.017564134 +0000 UTC m=+0.358355994 container start 122a96d4e4f399c20f58e25e5e7f68499dd6284f860bc818d7a7e142d7ed1f4f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-17T01:18:05.097 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:05 smithi100 podman[74730]: 2024-09-17 01:18:05.096996841 +0000 UTC m=+0.437788712 container attach 122a96d4e4f399c20f58e25e5e7f68499dd6284f860bc818d7a7e142d7ed1f4f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-09-17T01:18:05.477 INFO:journalctl@ceph.osd.7.smithi100.stdout:Sep 17 01:18:05 smithi100 podman[74730]: 2024-09-17 01:18:05.203573473 +0000 UTC m=+0.544365333 container died 122a96d4e4f399c20f58e25e5e7f68499dd6284f860bc818d7a7e142d7ed1f4f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-47eab686-748f-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-17T01:18:05.816 DEBUG:teuthology.orchestra.run.smithi100:> sudo pkill -f 'journalctl -f -n 0 -u ceph-47eab686-748f-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T01:18:05.856 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T01:18:05.857 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T01:18:05.857 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47eab686-748f-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T01:21:03.647 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47eab686-748f-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T01:21:23.362 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:23.398 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T01:21:23.426 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T01:21:23.583 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988/remote/smithi033/crash 2024-09-17T01:21:23.584 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/crash -- . 2024-09-17T01:21:23.633 INFO:teuthology.orchestra.run.smithi033.stderr:tar: /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T01:21:23.633 INFO:teuthology.orchestra.run.smithi033.stderr:tar: Error is not recoverable: exiting now 2024-09-17T01:21:23.634 DEBUG:teuthology.misc:Transferring archived files from smithi100:/var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988/remote/smithi100/crash 2024-09-17T01:21:23.635 DEBUG:teuthology.orchestra.run.smithi100:> sudo tar c -f - -C /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/crash -- . 2024-09-17T01:21:23.680 INFO:teuthology.orchestra.run.smithi100.stderr:tar: /var/lib/ceph/47eab686-748f-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T01:21:23.680 INFO:teuthology.orchestra.run.smithi100.stderr:tar: Error is not recoverable: exiting now 2024-09-17T01:21:23.681 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T01:21:23.681 DEBUG:teuthology.orchestra.run.smithi033:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-17T01:21:23.713 INFO:tasks.cephadm:Compressing logs... 2024-09-17T01:21:23.714 DEBUG:teuthology.orchestra.run.smithi033:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T01:21:23.756 DEBUG:teuthology.orchestra.run.smithi100:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T01:21:23.785 INFO:teuthology.orchestra.run.smithi100.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-09-17T01:21:23.785 INFO:teuthology.orchestra.run.smithi100.stderr:gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi100.ssfebu.log 2024-09-17T01:21:23.785 INFO:teuthology.orchestra.run.smithi100.stderr:gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-17T01:21:23.785 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi100.ssfebu.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T01:21:23.785 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mon.smithi100.log 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr: 88.3% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi100.ssfebu.log.gz 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mon.smithi100.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-17T01:21:23.786 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log 2024-09-17T01:21:23.787 INFO:teuthology.orchestra.run.smithi100.stderr: 85.6% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T01:21:23.787 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mgr.smithi100.qrwkie.log 2024-09-17T01:21:23.790 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log: 91.6% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T01:21:23.790 INFO:teuthology.orchestra.run.smithi100.stderr: 86.9% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T01:21:23.790 INFO:teuthology.orchestra.run.smithi100.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T01:21:23.792 INFO:teuthology.orchestra.run.smithi033.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-17T01:21:23.792 INFO:teuthology.orchestra.run.smithi033.stderr:: No such file or directory 2024-09-17T01:21:23.792 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mgr.smithi033.illmog.log 2024-09-17T01:21:23.792 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-17T01:21:23.792 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mgr.smithi033.illmog.log: /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.1.log: /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mon.smithi033.log 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mon.smithi033.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi033.gofagb.log 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.2.log: 85.6% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T01:21:23.793 INFO:teuthology.orchestra.run.smithi033.stderr: 91.5% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T01:21:23.794 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log 2024-09-17T01:21:23.794 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi033.gofagb.log: gzip -5 --verbose -- /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-17T01:21:23.795 INFO:teuthology.orchestra.run.smithi100.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mgr.smithi100.qrwkie.log: /var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T01:21:23.796 INFO:teuthology.orchestra.run.smithi100.stderr: 90.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T01:21:23.798 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log: 91.2% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T01:21:23.798 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T01:21:23.799 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.3.log: 86.9% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T01:21:23.820 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/cephadm.log: 91.3% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi033.gofagb.log.gz 2024-09-17T01:21:23.837 INFO:teuthology.orchestra.run.smithi033.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T01:21:23.857 INFO:teuthology.orchestra.run.smithi100.stderr: 89.7% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mgr.smithi100.qrwkie.log.gz 2024-09-17T01:21:24.054 INFO:teuthology.orchestra.run.smithi033.stderr: 88.7% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mgr.smithi033.illmog.log.gz 2024-09-17T01:21:24.229 INFO:teuthology.orchestra.run.smithi100.stderr: 92.6% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mon.smithi100.log.gz 2024-09-17T01:21:25.256 INFO:teuthology.orchestra.run.smithi033.stderr: 91.7% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-mon.smithi033.log.gz 2024-09-17T01:21:35.056 INFO:teuthology.orchestra.run.smithi033.stderr: 94.5% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-17T01:21:36.061 INFO:teuthology.orchestra.run.smithi100.stderr: 93.9% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-17T01:21:36.967 INFO:teuthology.orchestra.run.smithi100.stderr: 94.2% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-17T01:21:38.788 INFO:teuthology.orchestra.run.smithi033.stderr: 94.1% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-17T01:21:40.436 INFO:teuthology.orchestra.run.smithi033.stderr: 94.0% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-17T01:21:40.632 INFO:teuthology.orchestra.run.smithi033.stderr: 94.3% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-17T01:21:40.634 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-17T01:21:40.634 INFO:teuthology.orchestra.run.smithi033.stderr:real 0m16.863s 2024-09-17T01:21:40.634 INFO:teuthology.orchestra.run.smithi033.stderr:user 0m59.263s 2024-09-17T01:21:40.634 INFO:teuthology.orchestra.run.smithi033.stderr:sys 0m2.218s 2024-09-17T01:21:40.647 INFO:teuthology.orchestra.run.smithi100.stderr: 94.2% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-17T01:21:45.450 INFO:teuthology.orchestra.run.smithi100.stderr: 94.1% -- replaced with /var/log/ceph/47eab686-748f-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-17T01:21:45.451 INFO:teuthology.orchestra.run.smithi100.stderr: 2024-09-17T01:21:45.451 INFO:teuthology.orchestra.run.smithi100.stderr:real 0m21.681s 2024-09-17T01:21:45.452 INFO:teuthology.orchestra.run.smithi100.stderr:user 1m1.998s 2024-09-17T01:21:45.452 INFO:teuthology.orchestra.run.smithi100.stderr:sys 0m2.437s 2024-09-17T01:21:45.452 INFO:tasks.cephadm:Archiving logs... 2024-09-17T01:21:45.452 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988/remote/smithi033/log 2024-09-17T01:21:45.453 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T01:21:53.451 DEBUG:teuthology.misc:Transferring archived files from smithi100:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988/remote/smithi100/log 2024-09-17T01:21:53.452 DEBUG:teuthology.orchestra.run.smithi100:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T01:22:02.202 INFO:tasks.cephadm:Removing cluster... 2024-09-17T01:22:02.203 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47eab686-748f-11ef-bceb-c7b262605968 --force 2024-09-17T01:22:02.567 DEBUG:teuthology.orchestra.run.smithi100:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 47eab686-748f-11ef-bceb-c7b262605968 --force 2024-09-17T01:22:02.928 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T01:22:02.928 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T01:22:02.944 DEBUG:teuthology.orchestra.run.smithi100:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T01:22:02.960 INFO:tasks.cephadm:Teardown complete 2024-09-17T01:22:02.960 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T01:22:02.972 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T01:22:02.972 DEBUG:teuthology.orchestra.run.smithi033:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T01:22:02.987 DEBUG:teuthology.orchestra.run.smithi100:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T01:22:03.000 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-09-17T01:22:03.005 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T01:22:03.006 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-09-17T01:22:03.006 INFO:teuthology.orchestra.run.smithi033.stdout:^* hv01.front.sepia.ceph.com 3 6 377 14 +3164us[+3372us] +/- 43ms 2024-09-17T01:22:03.006 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 14 -3973us[-3765us] +/- 76ms 2024-09-17T01:22:03.006 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 12 -2300us[-2300us] +/- 96ms 2024-09-17T01:22:03.006 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:22:03.014 INFO:teuthology.orchestra.run.smithi100.stderr:bash: line 1: ntpq: command not found 2024-09-17T01:22:03.018 INFO:teuthology.orchestra.run.smithi100.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T01:22:03.018 INFO:teuthology.orchestra.run.smithi100.stdout:=============================================================================== 2024-09-17T01:22:03.019 INFO:teuthology.orchestra.run.smithi100.stdout:^* hv01.front.sepia.ceph.com 3 6 377 12 +3149us[+3114us] +/- 43ms 2024-09-17T01:22:03.019 INFO:teuthology.orchestra.run.smithi100.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 13 -3990us[-4026us] +/- 76ms 2024-09-17T01:22:03.019 INFO:teuthology.orchestra.run.smithi100.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 15 -2305us[-2341us] +/- 96ms 2024-09-17T01:22:03.019 INFO:teuthology.orchestra.run.smithi100.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T01:22:03.019 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T01:22:03.034 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T01:22:03.035 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T01:22:03.074 DEBUG:teuthology.orchestra.run.smithi033:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T01:22:03.157 DEBUG:teuthology.orchestra.run.smithi100:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T01:22:03.229 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-17T01:22:03.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:22:03.259 DEBUG:teuthology.orchestra.run.smithi100:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-17T01:22:03.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:22:03.287 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T01:22:03.302 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T01:22:03.354 INFO:teuthology.task.internal:Duration was 1932.963434 seconds 2024-09-17T01:22:03.355 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T01:22:03.388 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T01:22:03.388 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T01:22:03.391 DEBUG:teuthology.orchestra.run.smithi100:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T01:22:03.429 INFO:teuthology.orchestra.run.smithi100.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T01:22:03.429 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T01:22:03.904 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T01:22:03.905 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com 2024-09-17T01:22:03.905 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T01:22:03.967 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi100.front.sepia.ceph.com 2024-09-17T01:22:03.968 DEBUG:teuthology.orchestra.run.smithi100:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T01:22:03.994 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T01:22:03.994 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T01:22:04.010 DEBUG:teuthology.orchestra.run.smithi100:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T01:22:04.062 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T01:22:04.062 DEBUG:teuthology.orchestra.run.smithi033:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T01:22:04.079 DEBUG:teuthology.orchestra.run.smithi100:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T01:22:18.309 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T01:22:18.323 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T01:22:18.323 DEBUG:teuthology.orchestra.run.smithi033:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T01:22:18.349 DEBUG:teuthology.orchestra.run.smithi100:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T01:22:18.379 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T01:22:18.394 DEBUG:teuthology.orchestra.run.smithi033:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T01:22:18.397 DEBUG:teuthology.orchestra.run.smithi100:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T01:22:18.418 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = core 2024-09-17T01:22:18.447 INFO:teuthology.orchestra.run.smithi100.stdout:kernel.core_pattern = core 2024-09-17T01:22:18.705 DEBUG:teuthology.orchestra.run.smithi033:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T01:22:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:22:18.722 DEBUG:teuthology.orchestra.run.smithi100:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T01:22:18.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T01:22:18.763 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T01:22:18.776 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T01:22:18.777 DEBUG:teuthology.misc:Transferring archived files from smithi033:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988/remote/smithi033 2024-09-17T01:22:18.778 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T01:22:18.859 DEBUG:teuthology.misc:Transferring archived files from smithi100:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907988/remote/smithi100 2024-09-17T01:22:18.859 DEBUG:teuthology.orchestra.run.smithi100:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T01:22:18.904 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T01:22:18.904 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T01:22:18.907 DEBUG:teuthology.orchestra.run.smithi100:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T01:22:18.955 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T01:22:18.970 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T01:22:18.970 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T01:22:19.009 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T01:22:19.009 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T01:22:19.011 DEBUG:teuthology.orchestra.run.smithi100:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T01:22:19.025 INFO:teuthology.orchestra.run.smithi033.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 01:22 /home/ubuntu/cephtest 2024-09-17T01:22:19.028 INFO:teuthology.orchestra.run.smithi100.stdout: 265324 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 01:22 /home/ubuntu/cephtest 2024-09-17T01:22:19.029 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T01:22:19.071 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T01:22:19.157 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1932.963434457779 owner: scheduled_teuthology@teuthology success: true 2024-09-17T01:22:19.157 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T01:22:19.249 INFO:teuthology.run:pass