2024-09-16T23:42:17.108 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-16T23:42:17.128 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-16T23:42:17.216 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895 branch: squid description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: ceph-qa@ceph.com first_in_suite: false job_id: '7907895' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: squid sha1: e20bff7d16f5b09c58ead9548e8cf98cf87ee227 owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 599 sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f sleep_before_teardown: 0 subset: 33/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: e20bff7d16f5b09c58ead9548e8cf98cf87ee227 targets: smithi007.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEqGdRndLM57Bms68bmG596fx0SOaX92UlXFvxFMTcHfpwLTfU2ddE07yKwORNoe4axumWX0nCB0/ni/YV+ES8g= smithi194.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLzkGUQ5nvSvdd0MqEvBCydLcIQErtc+dJ7TReAiMmuJtDkzw6bu0ofVfTNW5ew97lGXFEJnLlyZy9au1qJWLbI= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-16_21:08:04 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-16T23:42:17.216 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa; will attempt to use it 2024-09-16T23:42:17.217 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_e20bff7d16f5b09c58ead9548e8cf98cf87ee227/qa/tasks 2024-09-16T23:42:17.217 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-16T23:42:17.219 INFO:teuthology.task.internal:Checking packages... 2024-09-16T23:42:17.241 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'c2ddf9721ef254645201cdbabd9c8f49c30a586f' 2024-09-16T23:42:17.241 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-16T23:42:17.241 INFO:teuthology.packaging:ref: None 2024-09-16T23:42:17.241 INFO:teuthology.packaging:tag: None 2024-09-16T23:42:17.241 INFO:teuthology.packaging:branch: squid 2024-09-16T23:42:17.241 INFO:teuthology.packaging:sha1: c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-16T23:42:17.241 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-09-16T23:42:17.451 INFO:teuthology.task.internal:Found packages for ceph version 19.2.0-0 2024-09-16T23:42:17.453 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-16T23:42:17.458 INFO:teuthology.task.internal:no buildpackages task found 2024-09-16T23:42:17.458 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-16T23:42:17.471 INFO:teuthology.task.internal:Saving configuration 2024-09-16T23:42:17.483 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-16T23:42:17.489 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-16T23:42:17.513 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi007.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-16 23:37:31.377332', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEqGdRndLM57Bms68bmG596fx0SOaX92UlXFvxFMTcHfpwLTfU2ddE07yKwORNoe4axumWX0nCB0/ni/YV+ES8g='} 2024-09-16T23:42:17.535 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi194.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-16 23:37:31.375778', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLzkGUQ5nvSvdd0MqEvBCydLcIQErtc+dJ7TReAiMmuJtDkzw6bu0ofVfTNW5ew97lGXFEJnLlyZy9au1qJWLbI='} 2024-09-16T23:42:17.536 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-16T23:42:17.541 INFO:teuthology.task.internal:roles: ubuntu@smithi007.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-16T23:42:17.542 INFO:teuthology.task.internal:roles: ubuntu@smithi194.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-16T23:42:17.542 INFO:teuthology.run_tasks:Running task console_log... 2024-09-16T23:42:17.600 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7ae790bac0>, signals=[15]) 2024-09-16T23:42:17.601 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-16T23:42:17.606 INFO:teuthology.task.internal:Opening connections... 2024-09-16T23:42:17.606 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:42:17.608 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi007.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:42:17.688 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi194.front.sepia.ceph.com 2024-09-16T23:42:17.689 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:42:17.762 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-16T23:42:17.769 DEBUG:teuthology.orchestra.run.smithi007:> uname -m 2024-09-16T23:42:17.788 INFO:teuthology.orchestra.run.smithi007.stdout:x86_64 2024-09-16T23:42:17.788 DEBUG:teuthology.orchestra.run.smithi007:> cat /etc/os-release 2024-09-16T23:42:17.845 INFO:teuthology.orchestra.run.smithi007.stdout:NAME="CentOS Stream" 2024-09-16T23:42:17.845 INFO:teuthology.orchestra.run.smithi007.stdout:VERSION="9" 2024-09-16T23:42:17.845 INFO:teuthology.orchestra.run.smithi007.stdout:ID="centos" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:ID_LIKE="rhel fedora" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:VERSION_ID="9" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:PLATFORM_ID="platform:el9" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:ANSI_COLOR="0;31" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:LOGO="fedora-logo-icon" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:HOME_URL="https://centos.org/" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-16T23:42:17.846 INFO:teuthology.orchestra.run.smithi007.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-16T23:42:17.847 INFO:teuthology.lock.ops:Updating smithi007.front.sepia.ceph.com on lock server 2024-09-16T23:42:17.866 DEBUG:teuthology.orchestra.run.smithi194:> uname -m 2024-09-16T23:42:17.885 INFO:teuthology.orchestra.run.smithi194.stdout:x86_64 2024-09-16T23:42:17.885 DEBUG:teuthology.orchestra.run.smithi194:> cat /etc/os-release 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:NAME="CentOS Stream" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION="9" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:ID="centos" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:ID_LIKE="rhel fedora" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:VERSION_ID="9" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:PLATFORM_ID="platform:el9" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:ANSI_COLOR="0;31" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:LOGO="fedora-logo-icon" 2024-09-16T23:42:17.943 INFO:teuthology.orchestra.run.smithi194.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-16T23:42:17.944 INFO:teuthology.orchestra.run.smithi194.stdout:HOME_URL="https://centos.org/" 2024-09-16T23:42:17.944 INFO:teuthology.orchestra.run.smithi194.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-16T23:42:17.944 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-16T23:42:17.944 INFO:teuthology.orchestra.run.smithi194.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-16T23:42:17.944 INFO:teuthology.lock.ops:Updating smithi194.front.sepia.ceph.com on lock server 2024-09-16T23:42:17.967 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-16T23:42:17.976 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-16T23:42:17.981 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-16T23:42:17.981 DEBUG:teuthology.orchestra.run.smithi007:> test '!' -e /home/ubuntu/cephtest 2024-09-16T23:42:17.983 DEBUG:teuthology.orchestra.run.smithi194:> test '!' -e /home/ubuntu/cephtest 2024-09-16T23:42:17.999 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-16T23:42:18.006 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-16T23:42:18.006 DEBUG:teuthology.orchestra.run.smithi007:> test -z $(ls -A /var/lib/ceph) 2024-09-16T23:42:18.041 DEBUG:teuthology.orchestra.run.smithi194:> test -z $(ls -A /var/lib/ceph) 2024-09-16T23:42:18.150 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-16T23:42:18.198 INFO:teuthology.run_tasks:Running task kernel... 2024-09-16T23:42:18.220 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-16T23:42:18.221 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-16T23:42:18.221 DEBUG:teuthology.orchestra.run.smithi007:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-16T23:42:18.221 DEBUG:teuthology.orchestra.run.smithi194:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-16T23:42:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T23:42:18.238 DEBUG:teuthology.orchestra.run.smithi007:> uname -r 2024-09-16T23:42:18.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T23:42:18.239 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-09-16T23:42:18.296 INFO:teuthology.orchestra.run.smithi007.stdout:5.14.0-503.el9.x86_64 2024-09-16T23:42:18.296 INFO:teuthology.task.kernel:Running kernel on smithi007: 5.14.0-503.el9.x86_64 2024-09-16T23:42:18.296 DEBUG:teuthology.orchestra.run.smithi007:> sudo yum install -y kernel 2024-09-16T23:42:18.298 INFO:teuthology.orchestra.run.smithi194.stdout:5.14.0-503.el9.x86_64 2024-09-16T23:42:18.298 INFO:teuthology.task.kernel:Running kernel on smithi194: 5.14.0-503.el9.x86_64 2024-09-16T23:42:18.298 DEBUG:teuthology.orchestra.run.smithi194:> sudo yum install -y kernel 2024-09-16T23:42:20.499 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - BaseOS 17 MB/s | 8.3 MB 00:00 2024-09-16T23:42:20.642 INFO:teuthology.orchestra.run.smithi007.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.3 MB 00:00 2024-09-16T23:42:22.393 INFO:teuthology.orchestra.run.smithi007.stdout:CentOS Stream 9 - AppStream 32 MB/s | 20 MB 00:00 2024-09-16T23:42:22.977 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - AppStream 14 MB/s | 20 MB 00:01 2024-09-16T23:42:26.875 INFO:teuthology.orchestra.run.smithi007.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-16T23:42:27.201 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.5 MB 00:00 2024-09-16T23:42:28.403 INFO:teuthology.orchestra.run.smithi007.stdout:CentOS Stream 9 - Extras packages 88 kB/s | 19 kB 00:00 2024-09-16T23:42:28.951 INFO:teuthology.orchestra.run.smithi194.stdout:CentOS Stream 9 - Extras packages 35 kB/s | 19 kB 00:00 2024-09-16T23:42:29.749 INFO:teuthology.orchestra.run.smithi007.stdout:Extra Packages for Enterprise Linux 19 MB/s | 23 MB 00:01 2024-09-16T23:42:29.764 INFO:teuthology.orchestra.run.smithi194.stdout:Extra Packages for Enterprise Linux 33 MB/s | 23 MB 00:00 2024-09-16T23:42:35.170 INFO:teuthology.orchestra.run.smithi194.stdout:lab-extras 37 kB/s | 1.7 kB 00:00 2024-09-16T23:42:35.346 INFO:teuthology.orchestra.run.smithi007.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-16T23:42:37.051 INFO:teuthology.orchestra.run.smithi194.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-16T23:42:37.051 INFO:teuthology.orchestra.run.smithi194.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-16T23:42:37.150 INFO:teuthology.orchestra.run.smithi194.stdout:Dependencies resolved. 2024-09-16T23:42:37.152 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-09-16T23:42:37.152 INFO:teuthology.orchestra.run.smithi194.stdout: Package Architecture Version Repository Size 2024-09-16T23:42:37.152 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout:Installing: 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout:Installing dependencies: 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction Summary 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout:================================================================================ 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout:Install 4 Packages 2024-09-16T23:42:37.153 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:42:37.154 INFO:teuthology.orchestra.run.smithi194.stdout:Total download size: 79 M 2024-09-16T23:42:37.154 INFO:teuthology.orchestra.run.smithi194.stdout:Installed size: 126 M 2024-09-16T23:42:37.154 INFO:teuthology.orchestra.run.smithi194.stdout:Downloading Packages: 2024-09-16T23:42:37.336 INFO:teuthology.orchestra.run.smithi007.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-16T23:42:37.336 INFO:teuthology.orchestra.run.smithi007.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-16T23:42:37.427 INFO:teuthology.orchestra.run.smithi007.stdout:Dependencies resolved. 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout:================================================================================ 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout: Package Architecture Version Repository Size 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout:================================================================================ 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout:Installing: 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout:Installing dependencies: 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-16T23:42:37.434 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout:Transaction Summary 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout:================================================================================ 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout:Install 4 Packages 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout:Total download size: 79 M 2024-09-16T23:42:37.435 INFO:teuthology.orchestra.run.smithi007.stdout:Installed size: 126 M 2024-09-16T23:42:37.436 INFO:teuthology.orchestra.run.smithi007.stdout:Downloading Packages: 2024-09-16T23:42:37.554 INFO:teuthology.orchestra.run.smithi194.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 297 kB/s | 36 kB 00:00 2024-09-16T23:42:38.281 INFO:teuthology.orchestra.run.smithi007.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 64 kB/s | 36 kB 00:00 2024-09-16T23:42:40.564 INFO:teuthology.orchestra.run.smithi007.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 12 MB/s | 35 MB 00:02 2024-09-16T23:42:41.439 INFO:teuthology.orchestra.run.smithi007.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 4.2 MB/s | 16 MB 00:03 2024-09-16T23:42:42.048 INFO:teuthology.orchestra.run.smithi007.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 7.6 MB/s | 29 MB 00:03 2024-09-16T23:42:42.048 INFO:teuthology.orchestra.run.smithi007.stdout:-------------------------------------------------------------------------------- 2024-09-16T23:42:42.049 INFO:teuthology.orchestra.run.smithi007.stdout:Total 17 MB/s | 79 MB 00:04 2024-09-16T23:42:42.754 INFO:teuthology.orchestra.run.smithi007.stdout:Running transaction check 2024-09-16T23:42:43.070 INFO:teuthology.orchestra.run.smithi194.stdout:(2/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 6.1 MB/s | 35 MB 00:05 2024-09-16T23:42:43.126 INFO:teuthology.orchestra.run.smithi007.stdout:Transaction check succeeded. 2024-09-16T23:42:43.126 INFO:teuthology.orchestra.run.smithi007.stdout:Running transaction test 2024-09-16T23:42:44.116 INFO:teuthology.orchestra.run.smithi007.stdout:Transaction test succeeded. 2024-09-16T23:42:44.117 INFO:teuthology.orchestra.run.smithi007.stdout:Running transaction 2024-09-16T23:42:44.170 INFO:teuthology.orchestra.run.smithi194.stdout:(3/4): kernel-core-5.14.0-508.el9.x86_64.rpm 2.3 MB/s | 16 MB 00:06 2024-09-16T23:42:46.696 INFO:teuthology.orchestra.run.smithi007.stdout: Preparing : 1/1 2024-09-16T23:42:47.369 INFO:teuthology.orchestra.run.smithi007.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-16T23:42:47.450 INFO:teuthology.orchestra.run.smithi007.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-16T23:42:50.217 INFO:teuthology.orchestra.run.smithi007.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-16T23:42:50.295 INFO:teuthology.orchestra.run.smithi194.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 2.2 MB/s | 29 MB 00:12 2024-09-16T23:42:50.295 INFO:teuthology.orchestra.run.smithi194.stdout:-------------------------------------------------------------------------------- 2024-09-16T23:42:50.295 INFO:teuthology.orchestra.run.smithi194.stdout:Total 6.0 MB/s | 79 MB 00:13 2024-09-16T23:42:50.343 INFO:teuthology.orchestra.run.smithi007.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-16T23:42:50.982 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction check 2024-09-16T23:42:51.326 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction check succeeded. 2024-09-16T23:42:51.327 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction test 2024-09-16T23:42:52.285 INFO:teuthology.orchestra.run.smithi194.stdout:Transaction test succeeded. 2024-09-16T23:42:52.285 INFO:teuthology.orchestra.run.smithi194.stdout:Running transaction 2024-09-16T23:42:54.721 INFO:teuthology.orchestra.run.smithi194.stdout: Preparing : 1/1 2024-09-16T23:42:55.358 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-16T23:42:55.445 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-16T23:42:55.463 INFO:teuthology.orchestra.run.smithi007.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-16T23:42:55.502 INFO:teuthology.orchestra.run.smithi007.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:42:58.194 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-16T23:42:58.331 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-16T23:43:00.640 INFO:teuthology.orchestra.run.smithi007.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:03.150 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-16T23:43:03.331 INFO:teuthology.orchestra.run.smithi194.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:08.161 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:39.992 INFO:teuthology.orchestra.run.smithi007.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:40.033 INFO:teuthology.orchestra.run.smithi007.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:40.402 INFO:teuthology.orchestra.run.smithi007.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:40.402 INFO:teuthology.orchestra.run.smithi007.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-16T23:43:40.402 INFO:teuthology.orchestra.run.smithi007.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-16T23:43:40.402 INFO:teuthology.orchestra.run.smithi007.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-16T23:43:41.313 INFO:teuthology.orchestra.run.smithi007.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout:Installed: 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:43:41.314 INFO:teuthology.orchestra.run.smithi007.stdout:Complete! 2024-09-16T23:43:41.649 DEBUG:teuthology.orchestra.run.smithi007:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T23:43:41.748 INFO:teuthology.orchestra.run.smithi007.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-16T23:43:41.748 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-16T23:43:41.748 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-16T23:43:41.749 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-16T23:43:41.749 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-16T23:43:41.749 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-16T23:43:41.749 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi007.front.sepia.ceph.com, path=None, version=distro) 2024-09-16T23:43:41.749 DEBUG:teuthology.orchestra.run.smithi007:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T23:43:41.808 INFO:teuthology.orchestra.run.smithi007.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-16T23:43:41.808 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-16T23:43:41.808 DEBUG:teuthology.orchestra.run.smithi007:> sudo rpm -qi grub2-tools 2024-09-16T23:43:41.844 INFO:teuthology.orchestra.run.smithi007.stdout:Name : grub2-tools 2024-09-16T23:43:41.844 INFO:teuthology.orchestra.run.smithi007.stdout:Epoch : 1 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Version : 2.06 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Release : 82.el9 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Architecture: x86_64 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Group : Unspecified 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Size : 8274670 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:License : GPLv3+ 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-16T23:43:41.845 INFO:teuthology.orchestra.run.smithi007.stdout:Packager : builder@centos.org 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:Vendor : CentOS 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:Summary : Support tools for GRUB. 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:Description : 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:hardware devices. 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:43:41.846 INFO:teuthology.orchestra.run.smithi007.stdout:This subpackage provides tools for support of all platforms. 2024-09-16T23:43:41.847 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-16T23:43:41.847 INFO:teuthology.task.kernel:Updating grub on smithi007 to boot 5.14.0-508.el9.x86_64 2024-09-16T23:43:41.847 DEBUG:teuthology.orchestra.run.smithi007:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-16T23:43:42.499 INFO:teuthology.orchestra.run.smithi007.stderr:Generating grub configuration file ... 2024-09-16T23:43:43.806 INFO:teuthology.orchestra.run.smithi007.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-16T23:43:43.827 INFO:teuthology.orchestra.run.smithi007.stderr:done 2024-09-16T23:43:43.835 DEBUG:teuthology.orchestra.run.smithi007:> mktemp 2024-09-16T23:43:43.850 INFO:teuthology.orchestra.run.smithi007.stdout:/tmp/tmp.w55BGzVFpo 2024-09-16T23:43:43.850 DEBUG:teuthology.orchestra.run.smithi007:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.w55BGzVFpo 2024-09-16T23:43:43.915 DEBUG:teuthology.orchestra.run.smithi007:> sudo chmod 0666 /tmp/tmp.w55BGzVFpo 2024-09-16T23:43:44.048 DEBUG:teuthology.orchestra.remote:smithi007:/tmp/tmp.w55BGzVFpo is 6KB 2024-09-16T23:43:44.097 DEBUG:teuthology.orchestra.run.smithi007:> rm -fr /tmp/tmp.w55BGzVFpo 2024-09-16T23:43:44.112 DEBUG:teuthology.orchestra.run.smithi007:> sudo /bin/ls /boot/loader/entries || true 2024-09-16T23:43:44.173 INFO:teuthology.orchestra.run.smithi007.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-16T23:43:44.174 INFO:teuthology.orchestra.run.smithi007.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-16T23:43:44.174 INFO:teuthology.orchestra.run.smithi007.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-16T23:43:44.174 INFO:teuthology.orchestra.run.smithi007.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-16T23:43:44.175 DEBUG:teuthology.orchestra.run.smithi007:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-16T23:43:44.334 DEBUG:teuthology.orchestra.run.smithi007:> sudo shutdown -r now 2024-09-16T23:43:46.190 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:46.235 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:46.571 INFO:teuthology.orchestra.run.smithi194.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:46.571 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-16T23:43:46.571 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-16T23:43:46.572 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-16T23:43:47.388 INFO:teuthology.orchestra.run.smithi194.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-16T23:43:47.388 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:43:47.388 INFO:teuthology.orchestra.run.smithi194.stdout:Installed: 2024-09-16T23:43:47.388 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-16T23:43:47.389 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-16T23:43:47.389 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-16T23:43:47.389 INFO:teuthology.orchestra.run.smithi194.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-16T23:43:47.389 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:43:47.389 INFO:teuthology.orchestra.run.smithi194.stdout:Complete! 2024-09-16T23:43:47.738 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T23:43:47.819 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-16T23:43:47.820 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-16T23:43:47.820 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-16T23:43:47.820 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-16T23:43:47.820 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-16T23:43:47.820 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-16T23:43:47.820 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi194.front.sepia.ceph.com, path=None, version=distro) 2024-09-16T23:43:47.820 DEBUG:teuthology.orchestra.run.smithi194:> rpm -q kernel | sort -rV | head -n 1 2024-09-16T23:43:47.878 INFO:teuthology.orchestra.run.smithi194.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-16T23:43:47.879 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-16T23:43:47.879 DEBUG:teuthology.orchestra.run.smithi194:> sudo rpm -qi grub2-tools 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Name : grub2-tools 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Epoch : 1 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Version : 2.06 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Release : 82.el9 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Architecture: x86_64 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Group : Unspecified 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Size : 8274670 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:License : GPLv3+ 2024-09-16T23:43:47.918 INFO:teuthology.orchestra.run.smithi194.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Packager : builder@centos.org 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Vendor : CentOS 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Summary : Support tools for GRUB. 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:Description : 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout:hardware devices. 2024-09-16T23:43:47.919 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:43:47.920 INFO:teuthology.orchestra.run.smithi194.stdout:This subpackage provides tools for support of all platforms. 2024-09-16T23:43:47.920 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-16T23:43:47.920 INFO:teuthology.task.kernel:Updating grub on smithi194 to boot 5.14.0-508.el9.x86_64 2024-09-16T23:43:47.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-16T23:43:48.595 INFO:teuthology.orchestra.run.smithi194.stderr:Generating grub configuration file ... 2024-09-16T23:43:49.879 INFO:teuthology.orchestra.run.smithi194.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-16T23:43:49.915 INFO:teuthology.orchestra.run.smithi194.stderr:done 2024-09-16T23:43:49.917 DEBUG:teuthology.orchestra.run.smithi194:> mktemp 2024-09-16T23:43:49.933 INFO:teuthology.orchestra.run.smithi194.stdout:/tmp/tmp.5DExPxridL 2024-09-16T23:43:49.933 DEBUG:teuthology.orchestra.run.smithi194:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.5DExPxridL 2024-09-16T23:43:50.004 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 0666 /tmp/tmp.5DExPxridL 2024-09-16T23:43:50.135 DEBUG:teuthology.orchestra.remote:smithi194:/tmp/tmp.5DExPxridL is 6KB 2024-09-16T23:43:50.184 DEBUG:teuthology.orchestra.run.smithi194:> rm -fr /tmp/tmp.5DExPxridL 2024-09-16T23:43:50.200 DEBUG:teuthology.orchestra.run.smithi194:> sudo /bin/ls /boot/loader/entries || true 2024-09-16T23:43:50.275 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-16T23:43:50.275 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-16T23:43:50.275 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-16T23:43:50.275 INFO:teuthology.orchestra.run.smithi194.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-16T23:43:50.276 DEBUG:teuthology.orchestra.run.smithi194:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-16T23:43:50.438 DEBUG:teuthology.orchestra.run.smithi194:> sudo shutdown -r now 2024-09-16T23:44:14.362 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-16T23:44:14.362 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi007.front.sepia.ceph.com' 2024-09-16T23:44:14.363 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi007.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:44:20.447 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-16T23:44:20.448 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-16T23:44:20.448 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:44:32.829 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.7 2024-09-16T23:44:38.973 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.194 2024-09-16T23:44:41.833 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi007.front.sepia.ceph.com' 2024-09-16T23:44:41.834 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi007.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:44:47.981 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-16T23:44:47.981 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:45:41.889 DEBUG:teuthology.orchestra.remote:timed out 2024-09-16T23:45:47.989 DEBUG:teuthology.orchestra.remote:timed out 2024-09-16T23:45:53.893 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi007.front.sepia.ceph.com' 2024-09-16T23:45:53.894 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi007.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:45:54.259 DEBUG:teuthology.orchestra.run.smithi007:> true 2024-09-16T23:45:54.631 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi007.front.sepia.ceph.com' 2024-09-16T23:45:54.631 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-508.el9.x86_64"... 2024-09-16T23:45:54.631 DEBUG:teuthology.orchestra.run.smithi007:> uname -r 2024-09-16T23:45:54.649 INFO:teuthology.orchestra.run.smithi007.stdout:5.14.0-508.el9.x86_64 2024-09-16T23:45:54.649 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-16T23:45:54.649 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-16T23:45:54.649 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-16T23:45:55.650 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-16T23:45:55.650 DEBUG:teuthology.orchestra.run.smithi007:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-16T23:45:55.740 INFO:teuthology.orchestra.run.smithi007.stdout:ttyS1 2024-09-16T23:45:55.757 DEBUG:teuthology.parallel:result is None 2024-09-16T23:45:59.990 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-16T23:45:59.991 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:46:00.446 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-09-16T23:46:00.781 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-16T23:46:00.781 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-508.el9.x86_64"... 2024-09-16T23:46:00.781 DEBUG:teuthology.orchestra.run.smithi194:> uname -r 2024-09-16T23:46:00.797 INFO:teuthology.orchestra.run.smithi194.stdout:5.14.0-508.el9.x86_64 2024-09-16T23:46:00.797 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-16T23:46:00.797 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-16T23:46:00.797 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-16T23:46:01.798 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-16T23:46:01.798 DEBUG:teuthology.orchestra.run.smithi194:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-16T23:46:01.867 INFO:teuthology.orchestra.run.smithi194.stdout:ttyS1 2024-09-16T23:46:01.887 DEBUG:teuthology.parallel:result is None 2024-09-16T23:46:01.887 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-16T23:46:01.894 INFO:teuthology.task.internal:Creating test directory... 2024-09-16T23:46:01.895 DEBUG:teuthology.orchestra.run.smithi007:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-16T23:46:01.897 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-16T23:46:01.915 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-16T23:46:01.921 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-16T23:46:01.928 INFO:teuthology.task.internal:Creating archive directory... 2024-09-16T23:46:01.928 DEBUG:teuthology.orchestra.run.smithi007:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-16T23:46:01.956 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-16T23:46:01.985 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-16T23:46:01.992 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-16T23:46:01.992 DEBUG:teuthology.orchestra.run.smithi007:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-16T23:46:02.024 DEBUG:teuthology.orchestra.run.smithi194:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-16T23:46:02.057 INFO:teuthology.orchestra.run.smithi007.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T23:46:02.058 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T23:46:02.069 INFO:teuthology.orchestra.run.smithi007.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T23:46:02.070 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-16T23:46:02.072 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-16T23:46:02.081 INFO:teuthology.task.internal:Configuring sudo... 2024-09-16T23:46:02.081 DEBUG:teuthology.orchestra.run.smithi007:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-16T23:46:02.114 DEBUG:teuthology.orchestra.run.smithi194:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-16T23:46:02.143 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-16T23:46:02.153 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-16T23:46:02.153 DEBUG:teuthology.orchestra.run.smithi007:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-16T23:46:02.183 DEBUG:teuthology.orchestra.run.smithi194:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-16T23:46:02.201 DEBUG:teuthology.orchestra.run.smithi007:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T23:46:02.289 DEBUG:teuthology.orchestra.run.smithi007:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T23:46:02.338 DEBUG:teuthology.orchestra.run.smithi007:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T23:46:02.425 DEBUG:teuthology.orchestra.run.smithi007:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T23:46:02.469 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:46:02.469 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-16T23:46:02.537 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T23:46:02.584 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-16T23:46:02.636 DEBUG:teuthology.orchestra.run.smithi194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T23:46:02.715 DEBUG:teuthology.orchestra.run.smithi194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-16T23:46:02.762 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-16T23:46:02.762 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-16T23:46:02.828 DEBUG:teuthology.orchestra.run.smithi007:> sudo service rsyslog restart 2024-09-16T23:46:02.831 DEBUG:teuthology.orchestra.run.smithi194:> sudo service rsyslog restart 2024-09-16T23:46:02.873 INFO:teuthology.orchestra.run.smithi007.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T23:46:02.913 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-16T23:46:03.385 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-16T23:46:03.393 INFO:teuthology.task.internal:Starting timer... 2024-09-16T23:46:03.393 INFO:teuthology.run_tasks:Running task pcp... 2024-09-16T23:46:03.415 INFO:teuthology.run_tasks:Running task selinux... 2024-09-16T23:46:03.425 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-09-16T23:46:03.426 DEBUG:teuthology.orchestra.run.smithi007:> sudo service auditd rotate 2024-09-16T23:46:03.479 INFO:teuthology.orchestra.run.smithi007.stdout:Rotating logs: 2024-09-16T23:46:03.481 DEBUG:teuthology.orchestra.run.smithi194:> sudo service auditd rotate 2024-09-16T23:46:03.531 INFO:teuthology.orchestra.run.smithi194.stdout:Rotating logs: 2024-09-16T23:46:03.533 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-16T23:46:03.533 DEBUG:teuthology.orchestra.run.smithi007:> /usr/sbin/getenforce 2024-09-16T23:46:03.563 INFO:teuthology.orchestra.run.smithi007.stdout:Permissive 2024-09-16T23:46:03.564 DEBUG:teuthology.orchestra.run.smithi194:> /usr/sbin/getenforce 2024-09-16T23:46:03.597 INFO:teuthology.orchestra.run.smithi194.stdout:Permissive 2024-09-16T23:46:03.598 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi007.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi194.front.sepia.ceph.com': 'permissive'} 2024-09-16T23:46:03.598 DEBUG:teuthology.orchestra.run.smithi007:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-16T23:46:03.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T23:46:03.632 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-16T23:46:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-16T23:46:03.665 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-16T23:46:03.665 DEBUG:teuthology.orchestra.run.smithi007:> sudo /usr/sbin/setenforce permissive 2024-09-16T23:46:03.702 DEBUG:teuthology.orchestra.run.smithi194:> sudo /usr/sbin/setenforce permissive 2024-09-16T23:46:03.741 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-16T23:46:03.750 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-16T23:46:03.762 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-16T23:46:03.763 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi007.front.sepia.ceph.com,smithi194.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-16T23:49:48.471 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi007.front.sepia.ceph.com'), Remote(name='ubuntu@smithi194.front.sepia.ceph.com')] 2024-09-16T23:49:48.473 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi007.front.sepia.ceph.com' 2024-09-16T23:49:48.474 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi007.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:49:48.554 DEBUG:teuthology.orchestra.run.smithi007:> true 2024-09-16T23:49:48.644 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi007.front.sepia.ceph.com' 2024-09-16T23:49:48.645 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-16T23:49:48.645 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-16T23:49:48.728 DEBUG:teuthology.orchestra.run.smithi194:> true 2024-09-16T23:49:48.808 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi194.front.sepia.ceph.com' 2024-09-16T23:49:48.808 INFO:teuthology.run_tasks:Running task clock... 2024-09-16T23:49:48.818 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-16T23:49:48.819 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-16T23:49:48.819 DEBUG:teuthology.orchestra.run.smithi007:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T23:49:48.822 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-16T23:49:48.822 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-16T23:49:48.851 INFO:teuthology.orchestra.run.smithi007.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-16T23:49:48.866 INFO:teuthology.orchestra.run.smithi007.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-16T23:49:48.877 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-16T23:49:48.891 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-16T23:49:48.893 INFO:teuthology.orchestra.run.smithi007.stderr:sudo: ntpd: command not found 2024-09-16T23:49:48.905 INFO:teuthology.orchestra.run.smithi007.stdout:506 Cannot talk to daemon 2024-09-16T23:49:48.915 INFO:teuthology.orchestra.run.smithi194.stderr:sudo: ntpd: command not found 2024-09-16T23:49:48.920 INFO:teuthology.orchestra.run.smithi007.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-16T23:49:48.926 INFO:teuthology.orchestra.run.smithi194.stdout:506 Cannot talk to daemon 2024-09-16T23:49:48.936 INFO:teuthology.orchestra.run.smithi007.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-16T23:49:48.940 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-16T23:49:48.954 INFO:teuthology.orchestra.run.smithi194.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-16T23:49:49.002 INFO:teuthology.orchestra.run.smithi007.stderr:bash: line 1: ntpq: command not found 2024-09-16T23:49:49.006 INFO:teuthology.orchestra.run.smithi007.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T23:49:49.007 INFO:teuthology.orchestra.run.smithi007.stdout:=============================================================================== 2024-09-16T23:49:49.007 INFO:teuthology.orchestra.run.smithi007.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.007 INFO:teuthology.orchestra.run.smithi007.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.007 INFO:teuthology.orchestra.run.smithi007.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.007 INFO:teuthology.orchestra.run.smithi007.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.013 INFO:teuthology.orchestra.run.smithi194.stderr:bash: line 1: ntpq: command not found 2024-09-16T23:49:49.017 INFO:teuthology.orchestra.run.smithi194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-16T23:49:49.017 INFO:teuthology.orchestra.run.smithi194.stdout:=============================================================================== 2024-09-16T23:49:49.017 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.017 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.017 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.017 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-16T23:49:49.018 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-16T23:49:49.114 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'c2ddf9721ef254645201cdbabd9c8f49c30a586f'} 2024-09-16T23:49:49.114 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-16T23:49:49.115 INFO:tasks.cephadm:Cluster fsid is 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:49:49.115 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-16T23:49:49.115 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-16T23:49:49.116 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi007': '172.21.15.7', 'mon.smithi194': '172.21.15.194'} 2024-09-16T23:49:49.116 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-16T23:49:49.116 DEBUG:teuthology.orchestra.run.smithi007:> sudo hostname $(hostname -s) 2024-09-16T23:49:49.143 DEBUG:teuthology.orchestra.run.smithi194:> sudo hostname $(hostname -s) 2024-09-16T23:49:49.170 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-16T23:49:49.170 DEBUG:teuthology.orchestra.run.smithi007:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-16T23:49:49.430 INFO:teuthology.orchestra.run.smithi007.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 16 23:49 /home/ubuntu/cephtest/cephadm 2024-09-16T23:49:49.431 DEBUG:teuthology.orchestra.run.smithi194:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-16T23:49:49.555 INFO:teuthology.orchestra.run.smithi194.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 16 23:49 /home/ubuntu/cephtest/cephadm 2024-09-16T23:49:49.556 DEBUG:teuthology.orchestra.run.smithi007:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-16T23:49:49.573 DEBUG:teuthology.orchestra.run.smithi194:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-16T23:49:49.597 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-16T23:49:49.598 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-16T23:49:49.615 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-16T23:49:51.232 INFO:teuthology.orchestra.run.smithi194.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T23:49:51.361 INFO:teuthology.orchestra.run.smithi007.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T23:50:03.530 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:50:03.530 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-16T23:50:03.530 INFO:teuthology.orchestra.run.smithi007.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-16T23:50:03.530 INFO:teuthology.orchestra.run.smithi007.stdout: "repo_digests": [ 2024-09-16T23:50:03.531 INFO:teuthology.orchestra.run.smithi007.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-16T23:50:03.531 INFO:teuthology.orchestra.run.smithi007.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-16T23:50:03.531 INFO:teuthology.orchestra.run.smithi007.stdout: ] 2024-09-16T23:50:03.531 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout:{ 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout: "repo_digests": [ 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout: ] 2024-09-16T23:50:03.722 INFO:teuthology.orchestra.run.smithi194.stdout:} 2024-09-16T23:50:03.740 DEBUG:teuthology.orchestra.run.smithi007:> sudo mkdir -p /etc/ceph 2024-09-16T23:50:03.770 DEBUG:teuthology.orchestra.run.smithi194:> sudo mkdir -p /etc/ceph 2024-09-16T23:50:03.801 DEBUG:teuthology.orchestra.run.smithi007:> sudo chmod 777 /etc/ceph 2024-09-16T23:50:03.836 DEBUG:teuthology.orchestra.run.smithi194:> sudo chmod 777 /etc/ceph 2024-09-16T23:50:03.866 INFO:tasks.cephadm:Writing seed config... 2024-09-16T23:50:03.867 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-16T23:50:03.867 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-16T23:50:03.867 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-16T23:50:03.867 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-16T23:50:03.867 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-16T23:50:03.868 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-16T23:50:03.868 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-16T23:50:03.868 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-16T23:50:03.868 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:50:03.868 DEBUG:teuthology.orchestra.run.smithi007:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-16T23:50:03.892 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 5c15e83c-7486-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-16T23:50:03.892 DEBUG:teuthology.orchestra.run.smithi007:mon.smithi007> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service 2024-09-16T23:50:03.935 INFO:tasks.cephadm:Bootstrapping... 2024-09-16T23:50:03.935 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.7 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-16T23:50:04.132 INFO:teuthology.orchestra.run.smithi007.stderr:-------------------------------------------------------------------------------- 2024-09-16T23:50:04.132 INFO:teuthology.orchestra.run.smithi007.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '5c15e83c-7486-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.7', '--skip-admin-label'] 2024-09-16T23:50:04.154 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: 5.2.2 2024-09-16T23:50:04.156 INFO:teuthology.orchestra.run.smithi007.stderr:Verifying podman|docker is present... 2024-09-16T23:50:04.177 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: 5.2.2 2024-09-16T23:50:04.179 INFO:teuthology.orchestra.run.smithi007.stderr:Verifying lvm2 is present... 2024-09-16T23:50:04.179 INFO:teuthology.orchestra.run.smithi007.stderr:Verifying time synchronization is in place... 2024-09-16T23:50:04.186 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-16T23:50:04.193 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: inactive 2024-09-16T23:50:04.200 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: enabled 2024-09-16T23:50:04.207 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: active 2024-09-16T23:50:04.207 INFO:teuthology.orchestra.run.smithi007.stderr:Unit chronyd.service is enabled and running 2024-09-16T23:50:04.207 INFO:teuthology.orchestra.run.smithi007.stderr:Repeating the final host check... 2024-09-16T23:50:04.227 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: 5.2.2 2024-09-16T23:50:04.229 INFO:teuthology.orchestra.run.smithi007.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-16T23:50:04.229 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl is present 2024-09-16T23:50:04.230 INFO:teuthology.orchestra.run.smithi007.stderr:lvcreate is present 2024-09-16T23:50:04.236 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-16T23:50:04.242 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: inactive 2024-09-16T23:50:04.249 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: enabled 2024-09-16T23:50:04.256 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: active 2024-09-16T23:50:04.257 INFO:teuthology.orchestra.run.smithi007.stderr:Unit chronyd.service is enabled and running 2024-09-16T23:50:04.257 INFO:teuthology.orchestra.run.smithi007.stderr:Host looks OK 2024-09-16T23:50:04.257 INFO:teuthology.orchestra.run.smithi007.stderr:Cluster fsid: 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:04.257 INFO:teuthology.orchestra.run.smithi007.stderr:Acquiring lock 139858639366032 on /run/cephadm/5c15e83c-7486-11ef-bceb-c7b262605968.lock 2024-09-16T23:50:04.257 INFO:teuthology.orchestra.run.smithi007.stderr:Lock 139858639366032 acquired on /run/cephadm/5c15e83c-7486-11ef-bceb-c7b262605968.lock 2024-09-16T23:50:04.257 INFO:teuthology.orchestra.run.smithi007.stderr:Verifying IP 172.21.15.7 port 3300 ... 2024-09-16T23:50:04.258 INFO:teuthology.orchestra.run.smithi007.stderr:Verifying IP 172.21.15.7 port 6789 ... 2024-09-16T23:50:04.258 INFO:teuthology.orchestra.run.smithi007.stderr:Base mon IP is 172.21.15.7, final addrv is [v2:172.21.15.7:3300,v1:172.21.15.7:6789] 2024-09-16T23:50:04.260 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.7 metric 100 2024-09-16T23:50:04.260 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.7 metric 100 2024-09-16T23:50:04.263 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-16T23:50:04.263 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-16T23:50:04.263 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1787sec hoplimit 64 pref medium 2024-09-16T23:50:04.266 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-16T23:50:04.266 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-16T23:50:04.266 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-16T23:50:04.266 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-16T23:50:04.266 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:15ea/64 scope link 2024-09-16T23:50:04.266 INFO:teuthology.orchestra.run.smithi007.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-16T23:50:04.267 INFO:teuthology.orchestra.run.smithi007.stderr:Mon IP `172.21.15.7` is in CIDR network `172.21.0.0/20` 2024-09-16T23:50:04.267 INFO:teuthology.orchestra.run.smithi007.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-16T23:50:04.267 INFO:teuthology.orchestra.run.smithi007.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-16T23:50:04.371 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-16T23:50:05.083 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Getting image source signatures 2024-09-16T23:50:05.083 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-16T23:50:05.083 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-16T23:50:05.083 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-16T23:50:05.083 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-16T23:50:05.084 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-16T23:50:05.084 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-16T23:50:05.258 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: Writing manifest to image destination 2024-09-16T23:50:05.427 INFO:teuthology.orchestra.run.smithi007.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-16T23:50:05.857 INFO:teuthology.orchestra.run.smithi007.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-16T23:50:06.359 INFO:teuthology.orchestra.run.smithi007.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-16T23:50:06.359 INFO:teuthology.orchestra.run.smithi007.stderr:Extracting ceph user uid/gid from container image... 2024-09-16T23:50:06.793 INFO:teuthology.orchestra.run.smithi007.stderr:stat: 167 167 2024-09-16T23:50:08.144 INFO:teuthology.orchestra.run.smithi007.stderr:Creating initial keys... 2024-09-16T23:50:11.947 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph-authtool: AQAyxOhmqT8LNhAA5A2CnIpxQJ7j7LPOiMuc9w== 2024-09-16T23:50:19.769 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph-authtool: AQA7xOhmavANKRAAH7fK1VvlcRjsFRHNajbI/g== 2024-09-16T23:50:20.674 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph-authtool: AQA8xOhmprpeIxAA8Nonrz5S7H8ws0XFIfp3Jg== 2024-09-16T23:50:21.265 INFO:teuthology.orchestra.run.smithi007.stderr:Creating initial monmap... 2024-09-16T23:50:21.664 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-16T23:50:21.664 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-16T23:50:21.664 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:21.664 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-16T23:50:22.287 INFO:teuthology.orchestra.run.smithi007.stderr:monmaptool for smithi007 [v2:172.21.15.7:3300,v1:172.21.15.7:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-16T23:50:22.288 INFO:teuthology.orchestra.run.smithi007.stderr:setting min_mon_release = octopus 2024-09-16T23:50:22.288 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/monmaptool: set fsid to 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:22.288 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-16T23:50:22.288 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:50:22.288 INFO:teuthology.orchestra.run.smithi007.stderr:Creating mon... 2024-09-16T23:50:23.218 INFO:teuthology.orchestra.run.smithi007.stderr:create mon.smithi007 on 2024-09-16T23:50:23.452 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-16T23:50:23.613 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-5c15e83c-7486-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5c15e83c-7486-11ef-bceb-c7b262605968.target. 2024-09-16T23:50:23.613 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-5c15e83c-7486-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-5c15e83c-7486-11ef-bceb-c7b262605968.target. 2024-09-16T23:50:23.929 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Failed to reset failed state of unit ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service: Unit ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service not loaded. 2024-09-16T23:50:23.937 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5c15e83c-7486-11ef-bceb-c7b262605968.target.wants/ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service → /etc/systemd/system/ceph-5c15e83c-7486-11ef-bceb-c7b262605968@.service. 2024-09-16T23:50:24.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:24 smithi007 podman[23350]: 2024-09-16 23:50:24.578491147 +0000 UTC m=+0.358025290 container start 5c369e760e3fa077197c16462004df8287ad3c9235562cd232631288607e1b41 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, version=8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.openshift.expose-services=, vendor=Red Hat, Inc.) 2024-09-16T23:50:24.669 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: disabled 2024-09-16T23:50:24.676 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: inactive 2024-09-16T23:50:24.677 INFO:teuthology.orchestra.run.smithi007.stderr:firewalld.service is not enabled 2024-09-16T23:50:24.677 INFO:teuthology.orchestra.run.smithi007.stderr:Not possible to enable service . firewalld.service is not available 2024-09-16T23:50:24.677 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for mon to start... 2024-09-16T23:50:24.678 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for mon... 2024-09-16T23:50:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:24 smithi007 ceph-mon[23377]: mkfs 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:24 smithi007 ceph-mon[23377]: mon.smithi007 is new leader, mons smithi007 in quorum (ranks 0) 2024-09-16T23:50:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:24 smithi007 bash[23350]: 5c369e760e3fa077197c16462004df8287ad3c9235562cd232631288607e1b41 2024-09-16T23:50:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:24 smithi007 systemd[1]: Started Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-16T23:50:25.485 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: cluster: 2024-09-16T23:50:25.485 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: id: 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:25.485 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-16T23:50:25.485 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:25.485 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: services: 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi007 (age 0.870031s) 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: data: 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: pgs: 2024-09-16T23:50:25.486 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:26.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:25 smithi007 ceph-mon[23377]: mon.smithi007 is new leader, mons smithi007 in quorum (ranks 0) 2024-09-16T23:50:26.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:25 smithi007 ceph-mon[23377]: monmap e1: 1 mons at {smithi007=[v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0]} 2024-09-16T23:50:26.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:25 smithi007 ceph-mon[23377]: fsmap 2024-09-16T23:50:26.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:25 smithi007 ceph-mon[23377]: osdmap e1: 0 total, 0 up, 0 in 2024-09-16T23:50:26.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:25 smithi007 ceph-mon[23377]: mgrmap e1: no daemons active 2024-09-16T23:50:26.021 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:25 smithi007 ceph-mon[23377]: from='client.? 172.21.15.7:0/1536607333' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-16T23:50:26.023 INFO:teuthology.orchestra.run.smithi007.stderr:mon is available 2024-09-16T23:50:26.023 INFO:teuthology.orchestra.run.smithi007.stderr:Assimilating anything we can from ceph.conf... 2024-09-16T23:50:26.549 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:26.549 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: [global] 2024-09-16T23:50:26.549 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: fsid = 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:26.549 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.7:3300,v1:172.21.15.7:6789] 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: [mgr] 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: [osd] 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-16T23:50:26.550 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-16T23:50:27.085 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:26 smithi007 ceph-mon[23377]: from='client.? 172.21.15.7:0/2725395022' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-16T23:50:27.085 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:26 smithi007 ceph-mon[23377]: from='client.? 172.21.15.7:0/2725395022' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-16T23:50:27.087 INFO:teuthology.orchestra.run.smithi007.stderr:Generating new minimal ceph.conf... 2024-09-16T23:50:27.848 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:27 smithi007 ceph-mon[23377]: from='client.? 172.21.15.7:0/3007766294' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:50:28.154 INFO:teuthology.orchestra.run.smithi007.stderr:Restarting the monitor... 2024-09-16T23:50:28.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:28 smithi007 systemd[1]: Stopping Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-16T23:50:28.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:28 smithi007 bash[23599]: Error: no container with name or ID "ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon.smithi007" found: no such container 2024-09-16T23:50:28.988 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:28 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[23365]: 2024-09-16T23:50:28.616+0000 7f5d38fc0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi007 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-16T23:50:28.988 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:28 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[23365]: 2024-09-16T23:50:28.616+0000 7f5d38fc0700 -1 mon.smithi007@0(leader) e1 *** Got Signal Terminated *** 2024-09-16T23:50:28.988 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:28 smithi007 podman[23611]: 2024-09-16 23:50:28.749825687 +0000 UTC m=+0.527240005 container died 5c369e760e3fa077197c16462004df8287ad3c9235562cd232631288607e1b41 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, GIT_CLEAN=True, release=754, vcs-type=git, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, RELEASE=HEAD, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream) 2024-09-16T23:50:29.246 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 podman[23611]: 2024-09-16 23:50:29.08136507 +0000 UTC m=+0.858779376 container cleanup 5c369e760e3fa077197c16462004df8287ad3c9235562cd232631288607e1b41 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, version=8, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, name=centos-stream, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux ) 2024-09-16T23:50:29.246 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 bash[23611]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007 2024-09-16T23:50:29.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 podman[23626]: 2024-09-16 23:50:29.249109446 +0000 UTC m=+0.498499443 container remove 5c369e760e3fa077197c16462004df8287ad3c9235562cd232631288607e1b41 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, name=centos-stream, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-16T23:50:29.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 bash[23640]: Error: no container with name or ID "ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon.smithi007" found: no such container 2024-09-16T23:50:29.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service: Deactivated successfully. 2024-09-16T23:50:29.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 systemd[1]: Stopped Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-16T23:50:29.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 systemd[1]: Starting Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 podman[23703]: 2024-09-16 23:50:29.516544126 +0000 UTC m=+0.100019025 container create 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, com.redhat.component=centos-stream-container, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, name=centos-stream, io.openshift.expose-services=, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 podman[23703]: 2024-09-16 23:50:29.433658355 +0000 UTC m=+0.017133209 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 podman[23703]: 2024-09-16 23:50:29.702504328 +0000 UTC m=+0.285979178 container init 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, ceph=True, RELEASE=HEAD, vcs-type=git, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, version=8, com.redhat.component=centos-stream-container, distribution-scope=public) 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 podman[23703]: 2024-09-16 23:50:29.705847749 +0000 UTC m=+0.289322594 container start 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, version=8, RELEASE=HEAD, ceph=True, GIT_CLEAN=True, com.redhat.component=centos-stream-container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, release=754, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: pidfile_write: ignore empty --pid-file 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: load: jerasure load: lrc 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: RocksDB version: 6.15.5 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-16T23:50:29.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Compile date Apr 18 2022 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: DB SUMMARY 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: DB Session ID: 61AJJ6LKCHVELJ910FP3 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: CURRENT file: CURRENT 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi007/store.db dir, Total Num: 1, files: 000008.sst 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi007/store.db: 000010.log size: 82687 ; 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.error_if_exists: 0 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.create_if_missing: 0 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.paranoid_checks: 1 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.env: 0x55ef31ac6860 2024-09-16T23:50:29.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.fs: Posix File System 2024-09-16T23:50:29.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.info_log: 0x55ef32a9dfa0 2024-09-16T23:50:29.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T23:50:29.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.statistics: (nil) 2024-09-16T23:50:29.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.use_fsync: 0 2024-09-16T23:50:29.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_log_file_size: 0 2024-09-16T23:50:29.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T23:50:29.778 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T23:50:29.778 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T23:50:29.778 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T23:50:29.778 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.allow_fallocate: 1 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.use_direct_reads: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.db_log_dir: 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi007/store.db 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T23:50:29.779 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.write_buffer_manager: 0x55ef32b8e330 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T23:50:29.780 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.unordered_write: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.row_cache: None 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.wal_filter: None 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.preserve_deletes: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.two_write_queues: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.atomic_flush: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T23:50:29.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.log_readahead_size: 0 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_background_jobs: 2 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_background_compactions: -1 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_subcompactions: 1 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T23:50:29.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_open_files: -1 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_background_flushes: -1 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Compression algorithms supported: 2024-09-16T23:50:29.783 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kZSTD supported: 0 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kXpressCompression supported: 0 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kLZ4Compression supported: 1 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kBZip2Compression supported: 0 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kZlibCompression supported: 1 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: kSnappyCompression supported: 1 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi007/store.db/MANIFEST-000009 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.merge_operator: 2024-09-16T23:50:29.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_filter: None 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ef32a69d00) 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: cache_index_and_filter_blocks: 1 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: pin_top_level_index_and_filter: 1 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: index_type: 0 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: data_block_index_type: 0 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: index_shortening: 1 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T23:50:29.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: hash_index_allow_collision: 1 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: checksum: 1 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: no_block_cache: 0 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache: 0x55ef32ad4170 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache_name: BinnedLRUCache 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache_options: 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: capacity : 536870912 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: num_shard_bits : 4 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: strict_capacity_limit : 0 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: high_pri_pool_ratio: 0.000 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache_compressed: (nil) 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: persistent_cache: (nil) 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_size: 4096 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_size_deviation: 10 2024-09-16T23:50:29.786 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_restart_interval: 16 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: index_block_restart_interval: 1 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: metadata_block_size: 4096 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: partition_filters: 0 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: use_delta_encoding: 1 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: whole_key_filtering: 1 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: verify_compression: 0 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: read_amp_bytes_per_bit: 0 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: format_version: 4 2024-09-16T23:50:29.787 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: enable_index_compression: 1 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_align: 0 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression: NoCompression 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.num_levels: 7 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T23:50:29.788 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T23:50:29.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T23:50:29.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.arena_block_size: 4194304 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T23:50:29.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.table_properties_collectors: 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.inplace_update_support: 0 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T23:50:29.792 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.bloom_locality: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.max_successive_merges: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.ttl: 2592000 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.enable_blob_files: false 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.min_blob_size: 0 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T23:50:29.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi007/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726530629733282, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726530629734056, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79369, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77633, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71942, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726530629, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "eed1488e-57c2-44bf-bd4e-1b7e71b80108", "db_session_id": "61AJJ6LKCHVELJ910FP3"}} 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726530629734406, "job": 1, "event": "recovery_finished"} 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi007/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55ef32abaa80 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: DB pointer 0x55ef32aca000 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-16T23:50:29.794 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** DB Stats ** 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** Compaction Stats [default] ** 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: L0 2/0 79.31 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Sum 2/0 79.31 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** Compaction Stats [default] ** 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative compaction: 0.00 GB write, 24.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:50:29.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval compaction: 0.00 GB write, 24.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** Compaction Stats [default] ** 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: L0 2/0 79.31 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Sum 2/0 79.31 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** Compaction Stats [default] ** 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:50:29.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 151.4 0.00 0.00 1 0.000 0 0 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative compaction: 0.00 GB write, 24.21 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T23:50:29.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: starting mon.smithi007 rank 0 at public addrs [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] at bind addrs [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi007 fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???) e1 preinit fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).mds e1 new map 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).mds e1 print_map 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: e1 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: legacy client fscid: -1 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: No filesystems configured 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T23:50:29.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-16T23:50:29.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 bash[23703]: 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff 2024-09-16T23:50:29.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 systemd[1]: Started Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-16T23:50:29.800 INFO:teuthology.orchestra.run.smithi007.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-16T23:50:30.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mon.smithi007 is new leader, mons smithi007 in quorum (ranks 0) 2024-09-16T23:50:30.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: monmap e1: 1 mons at {smithi007=[v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0]} 2024-09-16T23:50:30.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: fsmap 2024-09-16T23:50:30.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: osdmap e1: 0 total, 0 up, 0 in 2024-09-16T23:50:30.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:29 smithi007 ceph-mon[23721]: mgrmap e1: no daemons active 2024-09-16T23:50:30.868 INFO:teuthology.orchestra.run.smithi007.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-16T23:50:30.868 INFO:teuthology.orchestra.run.smithi007.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-16T23:50:30.868 INFO:teuthology.orchestra.run.smithi007.stderr:Creating mgr... 2024-09-16T23:50:30.869 INFO:teuthology.orchestra.run.smithi007.stderr:Verifying port 9283 ... 2024-09-16T23:50:31.057 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Failed to reset failed state of unit ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mgr.smithi007.czdphd.service: Unit ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mgr.smithi007.czdphd.service not loaded. 2024-09-16T23:50:31.064 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: Created symlink /etc/systemd/system/ceph-5c15e83c-7486-11ef-bceb-c7b262605968.target.wants/ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mgr.smithi007.czdphd.service → /etc/systemd/system/ceph-5c15e83c-7486-11ef-bceb-c7b262605968@.service. 2024-09-16T23:50:31.546 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:31 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2724479315' entity='client.admin' 2024-09-16T23:50:31.765 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: disabled 2024-09-16T23:50:31.773 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: inactive 2024-09-16T23:50:31.774 INFO:teuthology.orchestra.run.smithi007.stderr:firewalld.service is not enabled 2024-09-16T23:50:31.774 INFO:teuthology.orchestra.run.smithi007.stderr:Not possible to enable service . firewalld.service is not available 2024-09-16T23:50:31.783 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: disabled 2024-09-16T23:50:31.791 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: inactive 2024-09-16T23:50:31.792 INFO:teuthology.orchestra.run.smithi007.stderr:firewalld.service is not enabled 2024-09-16T23:50:31.792 INFO:teuthology.orchestra.run.smithi007.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-16T23:50:31.792 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for mgr to start... 2024-09-16T23:50:31.792 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for mgr... 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsid": "5c15e83c-7486-11ef-bceb-c7b262605968", 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "health": { 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T23:50:32.373 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 0 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "smithi007" 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T23:50:32.374 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:32.375 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T23:50:32.375 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T23:50:32.375 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T23:50:32.375 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T23:50:32.378 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T23:50:32.379 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "available": false, 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "iostat", 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "nfs", 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "restful" 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modified": "2024-09-16T23:50:24.616088+0000", 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:32.380 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T23:50:32.381 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:50:32.674 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:32 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2595186038' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T23:50:33.003 INFO:teuthology.orchestra.run.smithi007.stderr:mgr not available, waiting (1/15)... 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsid": "5c15e83c-7486-11ef-bceb-c7b262605968", 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "health": { 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.512 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 0 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "smithi007" 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.513 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T23:50:35.514 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:35.514 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T23:50:35.514 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T23:50:35.514 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T23:50:35.514 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T23:50:35.514 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T23:50:35.517 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T23:50:35.517 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.517 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T23:50:35.517 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T23:50:35.517 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.518 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "available": false, 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "iostat", 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "nfs", 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "restful" 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modified": "2024-09-16T23:50:24.616088+0000", 2024-09-16T23:50:35.519 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:35.520 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:35.520 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T23:50:35.520 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:50:36.076 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:35 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1244074186' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T23:50:36.193 INFO:teuthology.orchestra.run.smithi007.stderr:mgr not available, waiting (2/15)... 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: Activating manager daemon smithi007.czdphd 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: mgrmap e2: smithi007.czdphd(active, starting, since 0.0028863s) 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:50:38.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-16T23:50:38.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: Manager daemon smithi007.czdphd is now available 2024-09-16T23:50:38.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:50:38.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:50:38.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:38.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:38.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:38 smithi007 ceph-mon[23721]: from='mgr.14100 172.21.15.7:0/1319068396' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:38.689 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:38.689 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsid": "5c15e83c-7486-11ef-bceb-c7b262605968", 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "health": { 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 0 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "smithi007" 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:38.690 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T23:50:38.691 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T23:50:38.692 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "available": false, 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "iostat", 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "nfs", 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "restful" 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:38.693 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modified": "2024-09-16T23:50:24.616088+0000", 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T23:50:38.694 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:50:39.238 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:39 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1154845565' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T23:50:39.239 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:39 smithi007 ceph-mon[23721]: mgrmap e3: smithi007.czdphd(active, since 1.00634s) 2024-09-16T23:50:39.241 INFO:teuthology.orchestra.run.smithi007.stderr:mgr not available, waiting (3/15)... 2024-09-16T23:50:41.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:41 smithi007 ceph-mon[23721]: mgrmap e4: smithi007.czdphd(active, since 2s) 2024-09-16T23:50:41.958 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsid": "5c15e83c-7486-11ef-bceb-c7b262605968", 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "health": { 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "checks": {}, 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mutes": [] 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum": [ 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 0 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "smithi007" 2024-09-16T23:50:41.959 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "monmap": { 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osdmap": { 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-16T23:50:41.960 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgmap": { 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "fsmap": { 2024-09-16T23:50:41.961 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "available": true, 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modules": [ 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "iostat", 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "nfs", 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "restful" 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ], 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "servicemap": { 2024-09-16T23:50:41.962 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-16T23:50:41.963 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "modified": "2024-09-16T23:50:24.616088+0000", 2024-09-16T23:50:41.963 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "services": {} 2024-09-16T23:50:41.963 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: }, 2024-09-16T23:50:41.963 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-16T23:50:41.963 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:50:42.548 INFO:teuthology.orchestra.run.smithi007.stderr:mgr is available 2024-09-16T23:50:43.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:42 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2296098844' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: [global] 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: fsid = 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: [mgr] 2024-09-16T23:50:43.127 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-16T23:50:43.128 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 2024-09-16T23:50:43.128 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: [osd] 2024-09-16T23:50:43.128 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-16T23:50:43.128 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-16T23:50:43.697 INFO:teuthology.orchestra.run.smithi007.stderr:Enabling cephadm module... 2024-09-16T23:50:43.945 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:43 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/310498489' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-16T23:50:45.059 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:44 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1384889660' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-16T23:50:45.984 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:45 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1384889660' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-16T23:50:45.984 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:45 smithi007 ceph-mon[23721]: mgrmap e5: smithi007.czdphd(active, since 6s) 2024-09-16T23:50:45.985 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:50:45.985 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-16T23:50:45.986 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "available": true, 2024-09-16T23:50:45.986 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "active_name": "smithi007.czdphd", 2024-09-16T23:50:45.986 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-16T23:50:45.986 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:50:46.518 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for the mgr to restart... 2024-09-16T23:50:46.518 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for mgr epoch 5... 2024-09-16T23:50:46.767 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:46 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2321903364' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-16T23:50:52.167 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:50:52.167 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-16T23:50:52.167 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "initialized": true 2024-09-16T23:50:52.168 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: Active manager daemon smithi007.czdphd restarted 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: Activating manager daemon smithi007.czdphd 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: osdmap e2: 0 total, 0 up, 0 in 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: mgrmap e6: smithi007.czdphd(active, starting, since 0.0530484s) 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T23:50:52.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: Manager daemon smithi007.czdphd is now available 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:50:52.415 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:52 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:50:52.716 INFO:teuthology.orchestra.run.smithi007.stderr:mgr epoch 5 is available 2024-09-16T23:50:52.716 INFO:teuthology.orchestra.run.smithi007.stderr:Setting orchestrator backend to cephadm... 2024-09-16T23:50:53.344 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:53 smithi007 ceph-mon[23721]: mgrmap e7: smithi007.czdphd(active, since 1.05488s) 2024-09-16T23:50:53.344 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:53 smithi007 ceph-mon[23721]: [16/Sep/2024:23:50:52] ENGINE Bus STARTING 2024-09-16T23:50:53.344 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:53 smithi007 ceph-mon[23721]: [16/Sep/2024:23:50:52] ENGINE Serving on https://172.21.15.7:7150 2024-09-16T23:50:53.344 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:53 smithi007 ceph-mon[23721]: [16/Sep/2024:23:50:52] ENGINE Bus STARTED 2024-09-16T23:50:53.344 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:53 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:53.344 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:53 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:50:54.487 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: value unchanged 2024-09-16T23:50:54.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:54 smithi007 ceph-mon[23721]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:50:54.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:54 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:54.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:54 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:50:54.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:54 smithi007 ceph-mon[23721]: mgrmap e8: smithi007.czdphd(active, since 2s) 2024-09-16T23:50:55.122 INFO:teuthology.orchestra.run.smithi007.stderr:Generating ssh key... 2024-09-16T23:50:56.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:55 smithi007 ceph-mon[23721]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:50:56.995 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:56 smithi007 ceph-mon[23721]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:50:56.996 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:56 smithi007 ceph-mon[23721]: Generating ssh key... 2024-09-16T23:50:56.996 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:56 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:56.996 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:56 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:50:56.996 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCz0++ZJQXc53h2atXrxwPG/HdI3mD9QfpsnRIu0f1Q64/U8+F5PPJu3DhNboR+Lj5agpoVr/P9WWy2bK6gaX6HvjK78uOsq9WzmUYaMudKbPa+1yAKcM6U+liQss/WWZYoqI29LHNmtMXXKJmOMg2ECNCtdB8aIpyTmv2UcDNo7Sk9+wmHyfgs677ZSFVjUvhF6Lrx5dWoWl+W1IR4TM3efREhlYxzYoeNagq52sVIQ4WCAE43NqAD/4+RGKN9/ScaUG6jKMkI/X2uCWEll7eDvgBdoMuTEmXH3hWBsrPCTZV8jA6neB0pwzZFbivspikFCtTInh+UolVJJv4h5HNlbmM2JgnSmR5F+Hwy4wg1mT2VSryLj1oGR1cnGMSf9bBA2pUsbAjNrCoSXcKlPRqsHgw7D9S72DD5ar75BUEO/U+6mlK13ZDriue63RXHKlsOGyWXnO2CsJK9F4hST32Ul8Cr+vhtP3phJLXnL98nCNd+21xp0maXIHTDUG1FeAM= ceph-5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:50:57.594 INFO:teuthology.orchestra.run.smithi007.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-16T23:50:57.594 INFO:teuthology.orchestra.run.smithi007.stderr:Adding key to root@localhost authorized_keys... 2024-09-16T23:50:57.602 INFO:teuthology.orchestra.run.smithi007.stderr:Adding host smithi007... 2024-09-16T23:50:58.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:57 smithi007 ceph-mon[23721]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:50:59.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:58 smithi007 ceph-mon[23721]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi007", "addr": "172.21.15.7", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:50:59.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:50:58 smithi007 ceph-mon[23721]: Deploying cephadm binary to smithi007 2024-09-16T23:50:59.136 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Added host 'smithi007' with addr '172.21.15.7' 2024-09-16T23:50:59.736 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying mon service with default placement... 2024-09-16T23:51:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:00 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:00 smithi007 ceph-mon[23721]: Added host smithi007 2024-09-16T23:51:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:00 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:51:00.358 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-16T23:51:01.103 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying mgr service with default placement... 2024-09-16T23:51:01.672 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:01 smithi007 ceph-mon[23721]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:01.673 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:01 smithi007 ceph-mon[23721]: Saving service mon spec with placement count:5 2024-09-16T23:51:01.673 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:01 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:01.993 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-16T23:51:03.297 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:02 smithi007 ceph-mon[23721]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:03.297 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:02 smithi007 ceph-mon[23721]: Saving service mgr spec with placement count:2 2024-09-16T23:51:03.297 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:02 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:03.297 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:02 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:03.299 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying crash service with default placement... 2024-09-16T23:51:03.946 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-16T23:51:04.522 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying prometheus service with default placement... 2024-09-16T23:51:05.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:04 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:05.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:04 smithi007 ceph-mon[23721]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:05.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:04 smithi007 ceph-mon[23721]: Saving service crash spec with placement * 2024-09-16T23:51:05.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:04 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:05.159 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-16T23:51:05.971 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying grafana service with default placement... 2024-09-16T23:51:06.280 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:06 smithi007 ceph-mon[23721]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:06.280 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:06 smithi007 ceph-mon[23721]: Saving service prometheus spec with placement count:1 2024-09-16T23:51:06.280 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:06 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:06.591 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-16T23:51:07.529 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying node-exporter service with default placement... 2024-09-16T23:51:07.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:07 smithi007 ceph-mon[23721]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:07.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:07 smithi007 ceph-mon[23721]: Saving service grafana spec with placement count:1 2024-09-16T23:51:07.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:07 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:07.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:07 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:08.213 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-16T23:51:08.827 INFO:teuthology.orchestra.run.smithi007.stderr:Deploying alertmanager service with default placement... 2024-09-16T23:51:09.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:09 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:09.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:09 smithi007 ceph-mon[23721]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:09.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:09 smithi007 ceph-mon[23721]: Saving service node-exporter spec with placement * 2024-09-16T23:51:09.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:09 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:09.466 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-16T23:51:10.561 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:10 smithi007 ceph-mon[23721]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:10.561 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:10 smithi007 ceph-mon[23721]: Saving service alertmanager spec with placement count:1 2024-09-16T23:51:10.561 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:10 smithi007 ceph-mon[23721]: from='mgr.14120 172.21.15.7:0/925863388' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:11.710 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:11 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1501429474' entity='client.admin' 2024-09-16T23:51:12.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:12 smithi007 ceph-mon[23721]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:12.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:12 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/319193186' entity='client.admin' 2024-09-16T23:51:12.813 INFO:teuthology.orchestra.run.smithi007.stderr:Enabling the dashboard module... 2024-09-16T23:51:13.848 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:13 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/451551117' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-16T23:51:14.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:14 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/451551117' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-16T23:51:14.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:14 smithi007 ceph-mon[23721]: mgrmap e9: smithi007.czdphd(active, since 22s) 2024-09-16T23:51:14.868 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:51:14.869 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-16T23:51:14.869 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "available": true, 2024-09-16T23:51:14.869 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "active_name": "smithi007.czdphd", 2024-09-16T23:51:14.869 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-16T23:51:14.869 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:51:15.433 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for the mgr to restart... 2024-09-16T23:51:15.433 INFO:teuthology.orchestra.run.smithi007.stderr:Waiting for mgr epoch 9... 2024-09-16T23:51:15.957 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:15 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2131682437' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-16T23:51:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: Active manager daemon smithi007.czdphd restarted 2024-09-16T23:51:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: Activating manager daemon smithi007.czdphd 2024-09-16T23:51:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: osdmap e3: 0 total, 0 up, 0 in 2024-09-16T23:51:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: mgrmap e10: smithi007.czdphd(active, starting, since 0.744093s) 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: Manager daemon smithi007.czdphd is now available 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:51:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:21 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:51:21.755 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: { 2024-09-16T23:51:21.755 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-16T23:51:21.755 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: "initialized": true 2024-09-16T23:51:21.755 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: } 2024-09-16T23:51:22.525 INFO:teuthology.orchestra.run.smithi007.stderr:mgr epoch 9 is available 2024-09-16T23:51:22.525 INFO:teuthology.orchestra.run.smithi007.stderr:Generating a dashboard self-signed certificate... 2024-09-16T23:51:22.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:22 smithi007 ceph-mon[23721]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-16T23:51:22.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:22 smithi007 ceph-mon[23721]: mgrmap e11: smithi007.czdphd(active, since 1.7465s) 2024-09-16T23:51:22.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:22 smithi007 ceph-mon[23721]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-16T23:51:22.796 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:22 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:23.279 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-16T23:51:23.874 INFO:teuthology.orchestra.run.smithi007.stderr:Creating initial admin user... 2024-09-16T23:51:24.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: mgrmap e12: smithi007.czdphd(active, since 2s) 2024-09-16T23:51:24.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:24.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: [16/Sep/2024:23:51:23] ENGINE Bus STARTING 2024-09-16T23:51:24.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:24.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:24.014 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: [16/Sep/2024:23:51:23] ENGINE Serving on https://172.21.15.7:7150 2024-09-16T23:51:24.014 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: [16/Sep/2024:23:51:23] ENGINE Bus STARTED 2024-09-16T23:51:24.014 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:23 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:24.798 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$ioSbB2LYuRmScL1thHo6Ze.qaNNFDSC3C/2cjrOxpoA7UHHejCvNu", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726530684, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-16T23:51:24.939 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:24 smithi007 ceph-mon[23721]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:25.517 INFO:teuthology.orchestra.run.smithi007.stderr:Fetching dashboard port number... 2024-09-16T23:51:26.086 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:25 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:26.145 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: 8443 2024-09-16T23:51:27.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:26 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2744875779' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-16T23:51:27.113 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: disabled 2024-09-16T23:51:27.122 INFO:teuthology.orchestra.run.smithi007.stderr:systemctl: inactive 2024-09-16T23:51:27.123 INFO:teuthology.orchestra.run.smithi007.stderr:firewalld.service is not enabled 2024-09-16T23:51:27.123 INFO:teuthology.orchestra.run.smithi007.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-16T23:51:27.125 INFO:teuthology.orchestra.run.smithi007.stderr:Ceph Dashboard is now available at: 2024-09-16T23:51:27.125 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:27.126 INFO:teuthology.orchestra.run.smithi007.stderr: URL: https://smithi007.front.sepia.ceph.com:8443/ 2024-09-16T23:51:27.126 INFO:teuthology.orchestra.run.smithi007.stderr: User: admin 2024-09-16T23:51:27.126 INFO:teuthology.orchestra.run.smithi007.stderr: Password: n3zs88vpwj 2024-09-16T23:51:27.126 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:27.126 INFO:teuthology.orchestra.run.smithi007.stderr:Enabling autotune for osd_memory_target 2024-09-16T23:51:28.055 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:27 smithi007 ceph-mon[23721]: mgrmap e13: smithi007.czdphd(active, since 6s) 2024-09-16T23:51:29.142 INFO:teuthology.orchestra.run.smithi007.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr:You can access the Ceph CLI with: 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr: ceph telemetry on 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:29.773 INFO:teuthology.orchestra.run.smithi007.stderr:For more information see: 2024-09-16T23:51:29.774 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:29.774 INFO:teuthology.orchestra.run.smithi007.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-16T23:51:29.774 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-16T23:51:29.774 INFO:teuthology.orchestra.run.smithi007.stderr:Bootstrap complete. 2024-09-16T23:51:29.812 INFO:tasks.cephadm:Fetching config... 2024-09-16T23:51:29.813 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:51:29.813 DEBUG:teuthology.orchestra.run.smithi007:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-16T23:51:29.831 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-16T23:51:29.831 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:51:29.831 DEBUG:teuthology.orchestra.run.smithi007:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-16T23:51:29.888 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-16T23:51:29.888 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:51:29.888 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd if=/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/keyring of=/dev/stdout 2024-09-16T23:51:29.954 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-16T23:51:29.954 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:51:29.954 DEBUG:teuthology.orchestra.run.smithi007:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-16T23:51:30.011 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-16T23:51:30.011 DEBUG:teuthology.orchestra.run.smithi007:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCz0++ZJQXc53h2atXrxwPG/HdI3mD9QfpsnRIu0f1Q64/U8+F5PPJu3DhNboR+Lj5agpoVr/P9WWy2bK6gaX6HvjK78uOsq9WzmUYaMudKbPa+1yAKcM6U+liQss/WWZYoqI29LHNmtMXXKJmOMg2ECNCtdB8aIpyTmv2UcDNo7Sk9+wmHyfgs677ZSFVjUvhF6Lrx5dWoWl+W1IR4TM3efREhlYxzYoeNagq52sVIQ4WCAE43NqAD/4+RGKN9/ScaUG6jKMkI/X2uCWEll7eDvgBdoMuTEmXH3hWBsrPCTZV8jA6neB0pwzZFbivspikFCtTInh+UolVJJv4h5HNlbmM2JgnSmR5F+Hwy4wg1mT2VSryLj1oGR1cnGMSf9bBA2pUsbAjNrCoSXcKlPRqsHgw7D9S72DD5ar75BUEO/U+6mlK13ZDriue63RXHKlsOGyWXnO2CsJK9F4hST32Ul8Cr+vhtP3phJLXnL98nCNd+21xp0maXIHTDUG1FeAM= ceph-5c15e83c-7486-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-16T23:51:30.086 INFO:teuthology.orchestra.run.smithi007.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCz0++ZJQXc53h2atXrxwPG/HdI3mD9QfpsnRIu0f1Q64/U8+F5PPJu3DhNboR+Lj5agpoVr/P9WWy2bK6gaX6HvjK78uOsq9WzmUYaMudKbPa+1yAKcM6U+liQss/WWZYoqI29LHNmtMXXKJmOMg2ECNCtdB8aIpyTmv2UcDNo7Sk9+wmHyfgs677ZSFVjUvhF6Lrx5dWoWl+W1IR4TM3efREhlYxzYoeNagq52sVIQ4WCAE43NqAD/4+RGKN9/ScaUG6jKMkI/X2uCWEll7eDvgBdoMuTEmXH3hWBsrPCTZV8jA6neB0pwzZFbivspikFCtTInh+UolVJJv4h5HNlbmM2JgnSmR5F+Hwy4wg1mT2VSryLj1oGR1cnGMSf9bBA2pUsbAjNrCoSXcKlPRqsHgw7D9S72DD5ar75BUEO/U+6mlK13ZDriue63RXHKlsOGyWXnO2CsJK9F4hST32Ul8Cr+vhtP3phJLXnL98nCNd+21xp0maXIHTDUG1FeAM= ceph-5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:51:30.098 DEBUG:teuthology.orchestra.run.smithi194:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCz0++ZJQXc53h2atXrxwPG/HdI3mD9QfpsnRIu0f1Q64/U8+F5PPJu3DhNboR+Lj5agpoVr/P9WWy2bK6gaX6HvjK78uOsq9WzmUYaMudKbPa+1yAKcM6U+liQss/WWZYoqI29LHNmtMXXKJmOMg2ECNCtdB8aIpyTmv2UcDNo7Sk9+wmHyfgs677ZSFVjUvhF6Lrx5dWoWl+W1IR4TM3efREhlYxzYoeNagq52sVIQ4WCAE43NqAD/4+RGKN9/ScaUG6jKMkI/X2uCWEll7eDvgBdoMuTEmXH3hWBsrPCTZV8jA6neB0pwzZFbivspikFCtTInh+UolVJJv4h5HNlbmM2JgnSmR5F+Hwy4wg1mT2VSryLj1oGR1cnGMSf9bBA2pUsbAjNrCoSXcKlPRqsHgw7D9S72DD5ar75BUEO/U+6mlK13ZDriue63RXHKlsOGyWXnO2CsJK9F4hST32Ul8Cr+vhtP3phJLXnL98nCNd+21xp0maXIHTDUG1FeAM= ceph-5c15e83c-7486-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-16T23:51:30.218 INFO:teuthology.orchestra.run.smithi194.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCz0++ZJQXc53h2atXrxwPG/HdI3mD9QfpsnRIu0f1Q64/U8+F5PPJu3DhNboR+Lj5agpoVr/P9WWy2bK6gaX6HvjK78uOsq9WzmUYaMudKbPa+1yAKcM6U+liQss/WWZYoqI29LHNmtMXXKJmOMg2ECNCtdB8aIpyTmv2UcDNo7Sk9+wmHyfgs677ZSFVjUvhF6Lrx5dWoWl+W1IR4TM3efREhlYxzYoeNagq52sVIQ4WCAE43NqAD/4+RGKN9/ScaUG6jKMkI/X2uCWEll7eDvgBdoMuTEmXH3hWBsrPCTZV8jA6neB0pwzZFbivspikFCtTInh+UolVJJv4h5HNlbmM2JgnSmR5F+Hwy4wg1mT2VSryLj1oGR1cnGMSf9bBA2pUsbAjNrCoSXcKlPRqsHgw7D9S72DD5ar75BUEO/U+6mlK13ZDriue63RXHKlsOGyWXnO2CsJK9F4hST32Ul8Cr+vhtP3phJLXnL98nCNd+21xp0maXIHTDUG1FeAM= ceph-5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:51:30.232 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-16T23:51:30.348 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:30 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3144438487' entity='client.admin' 2024-09-16T23:51:33.199 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:32 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:33.199 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:32 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi007", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:51:33.199 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:32 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:33.199 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:32 smithi007 ceph-mon[23721]: Deploying daemon alertmanager.smithi007 on smithi007 2024-09-16T23:51:34.121 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-16T23:51:34.122 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-16T23:51:34.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:34 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3247362868' entity='client.admin' 2024-09-16T23:51:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:37 smithi007 ceph-mon[23721]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:37 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:38.493 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi194 2024-09-16T23:51:38.493 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-16T23:51:38.494 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/etc/ceph/ceph.conf 2024-09-16T23:51:38.512 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-16T23:51:38.512 DEBUG:teuthology.orchestra.run.smithi194:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:51:38.571 INFO:tasks.cephadm:Adding host smithi194 to orchestrator... 2024-09-16T23:51:38.571 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch host add smithi194 2024-09-16T23:51:42.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:41 smithi007 ceph-mon[23721]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:42.941 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:42 smithi007 ceph-mon[23721]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi194", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:43.403 INFO:teuthology.orchestra.run.smithi007.stdout:Added host 'smithi194' with addr '172.21.15.194' 2024-09-16T23:51:44.402 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: Deploying cephadm binary to smithi194 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: Added host smithi194 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T23:51:44.677 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:44 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:51:45.747 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:45 smithi007 ceph-mon[23721]: Deploying daemon crash.smithi007 on smithi007 2024-09-16T23:51:46.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:46 smithi007 ceph-mon[23721]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:46.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:46 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:47.399 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:51:47.399 INFO:teuthology.orchestra.run.smithi007.stdout:[{"addr": "172.21.15.7", "hostname": "smithi007", "labels": [], "status": ""}, {"addr": "172.21.15.194", "hostname": "smithi194", "labels": [], "status": ""}] 2024-09-16T23:51:48.214 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-16T23:51:48.215 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T23:51:48.539 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:48 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:49.763 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:49 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T23:51:49.764 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:49 smithi007 ceph-mon[23721]: Deploying daemon grafana.smithi007 on smithi007 2024-09-16T23:51:50.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:50 smithi007 ceph-mon[23721]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:51.414 INFO:teuthology.orchestra.run.smithi007.stderr:adjusted tunables profile to default 2024-09-16T23:51:51.821 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:51 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/267615095' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-16T23:51:51.822 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:51 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:52.111 INFO:tasks.cephadm:Adding mon.smithi007 on smithi007 2024-09-16T23:51:52.111 INFO:tasks.cephadm:Adding mon.smithi194 on smithi194 2024-09-16T23:51:52.111 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi007:172.21.15.7=smithi007;smithi194:172.21.15.194=smithi194' 2024-09-16T23:51:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:52 smithi007 ceph-mon[23721]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:52 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/267615095' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-16T23:51:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:52 smithi007 ceph-mon[23721]: osdmap e4: 0 total, 0 up, 0 in 2024-09-16T23:51:53.907 INFO:teuthology.orchestra.run.smithi194.stdout:Scheduled mon update... 2024-09-16T23:51:54.450 DEBUG:teuthology.orchestra.run.smithi194:mon.smithi194> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi194.service 2024-09-16T23:51:54.453 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:51:54.453 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:51:54.848 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:54 smithi007 ceph-mon[23721]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:54 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:55 smithi007 ceph-mon[23721]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi007:172.21.15.7=smithi007;smithi194:172.21.15.194=smithi194", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:51:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:55 smithi007 ceph-mon[23721]: Saving service mon spec with placement smithi007:172.21.15.7=smithi007;smithi194:172.21.15.194=smithi194;count:2 2024-09-16T23:51:56.297 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:51:56.297 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:51:56.299 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:51:56.756 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:56 smithi007 ceph-mon[23721]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:56.756 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:56 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/119696515' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:51:57.787 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:51:57.787 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:51:58.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:58 smithi007 ceph-mon[23721]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:59.601 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:51:59.601 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:51:59.603 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:51:59.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:59 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:51:59.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:59 smithi007 ceph-mon[23721]: Deploying daemon node-exporter.smithi007 on smithi007 2024-09-16T23:51:59.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:59 smithi007 ceph-mon[23721]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:51:59.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:51:59 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/2338446615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:01.196 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:01.197 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:01 smithi007 ceph-mon[23721]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:01 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:01 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:01 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-16T23:52:02.965 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:02.965 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:02.967 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:03.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:03 smithi007 ceph-mon[23721]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-16T23:52:03.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:03 smithi007 ceph-mon[23721]: mgrmap e14: smithi007.czdphd(active, since 42s) 2024-09-16T23:52:03.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:03 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/3405728631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:04.490 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:04.490 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:06.263 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:06.264 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:06.266 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:06.590 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:06 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/1916497105' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:07.793 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:07.793 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:09.646 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:09.646 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:09.648 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: Active manager daemon smithi007.czdphd restarted 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: Activating manager daemon smithi007.czdphd 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: mgrmap e15: smithi007.czdphd(active, starting, since 0.0523531s) 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T23:52:10.072 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: Manager daemon smithi007.czdphd is now available 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/321670129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:10.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:11.135 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:11.136 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:11.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:11 smithi007 ceph-mon[23721]: [16/Sep/2024:23:52:09] ENGINE Bus STARTING 2024-09-16T23:52:11.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:11 smithi007 ceph-mon[23721]: [16/Sep/2024:23:52:09] ENGINE Serving on https://172.21.15.7:7150 2024-09-16T23:52:11.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:11 smithi007 ceph-mon[23721]: [16/Sep/2024:23:52:09] ENGINE Bus STARTED 2024-09-16T23:52:11.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:11 smithi007 ceph-mon[23721]: mgrmap e16: smithi007.czdphd(active, since 1.05476s) 2024-09-16T23:52:11.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:11 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:12.547 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:12 smithi007 ceph-mon[23721]: mgrmap e17: smithi007.czdphd(active, since 2s) 2024-09-16T23:52:12.547 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:12 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:13.732 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:13.733 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:13.733 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi007", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:52:13.733 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:14.380 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:14.380 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:14.381 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:14.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:14 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-16T23:52:14.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:14 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:52:14.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:14 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/2958478254' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:16.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi194", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:16.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:15 smithi007 ceph-mon[23721]: Deploying daemon crash.smithi194 on smithi194 2024-09-16T23:52:16.419 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:16.419 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:18.817 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:18.817 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:18.818 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/1324014835' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:20.372 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:20.372 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:20 smithi007 ceph-mon[23721]: Deploying daemon mgr.smithi194.xlteft on smithi194 2024-09-16T23:52:22.365 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:22.366 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:22.367 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:22.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:22 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/2965647238' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:23.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:23.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T23:52:23.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:23.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:23 smithi007 ceph-mon[23721]: Deploying daemon mon.smithi194 on smithi194 2024-09-16T23:52:24.117 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:24.117 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:25.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:25 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:26.868 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:26.868 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":1,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:50:21.589767Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-16T23:52:26.870 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 1 2024-09-16T23:52:27.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:26 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/250922313' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.654 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 podman[25670]: 2024-09-16 23:52:27.636335916 +0000 UTC m=+0.922839404 container start 32c4fa431c9c102b08ccac3d9ff3a24e0fa1ada9ec0e0a9f7e08cdf7da2d5171 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: set uid:gid to 167:167 (ceph:ceph) 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pidfile_write: ignore empty --pid-file 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: load: jerasure load: lrc 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: RocksDB version: 6.15.5 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-16T23:52:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Compile date Apr 18 2022 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: DB SUMMARY 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: DB Session ID: L9TN3VBEDRLLUWBMOEQB 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: CURRENT file: CURRENT 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: IDENTITY file: IDENTITY 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi194/store.db dir, Total Num: 0, files: 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi194/store.db: 000004.log size: 511 ; 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.error_if_exists: 0 2024-09-16T23:52:27.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.create_if_missing: 0 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.paranoid_checks: 1 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.env: 0x5572db8c6860 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.fs: Posix File System 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.info_log: 0x5572dd8f7fa0 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_file_opening_threads: 16 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.statistics: (nil) 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.use_fsync: 0 2024-09-16T23:52:27.912 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_log_file_size: 0 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.keep_log_file_num: 1000 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.recycle_log_file_num: 0 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.allow_fallocate: 1 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.allow_mmap_reads: 0 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.allow_mmap_writes: 0 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.use_direct_reads: 0 2024-09-16T23:52:27.913 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-16T23:52:27.914 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.create_missing_column_families: 0 2024-09-16T23:52:27.914 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.db_log_dir: 2024-09-16T23:52:27.914 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi194/store.db 2024-09-16T23:52:27.914 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-16T23:52:27.914 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-16T23:52:27.916 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.advise_random_on_open: 1 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.db_write_buffer_size: 0 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.write_buffer_manager: 0x5572dd9e8330 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-16T23:52:27.917 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.rate_limiter: (nil) 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.wal_recovery_mode: 2 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.enable_thread_tracking: 0 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.enable_pipelined_write: 0 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.unordered_write: 0 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-16T23:52:27.918 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.row_cache: None 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.wal_filter: None 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.allow_ingest_behind: 0 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.preserve_deletes: 0 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.two_write_queues: 0 2024-09-16T23:52:27.919 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.manual_wal_flush: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.atomic_flush: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.log_readahead_size: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.best_efforts_recovery: 0 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-16T23:52:27.920 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.allow_data_in_errors: 0 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.db_host_id: __hostname__ 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_background_jobs: 2 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_background_compactions: -1 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_subcompactions: 1 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_total_wal_size: 0 2024-09-16T23:52:27.921 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_open_files: -1 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bytes_per_sync: 0 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_readahead_size: 0 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_background_flushes: -1 2024-09-16T23:52:27.922 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Compression algorithms supported: 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kZSTD supported: 0 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kXpressCompression supported: 0 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kLZ4HCCompression supported: 1 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kLZ4Compression supported: 1 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kBZip2Compression supported: 0 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kZlibCompression supported: 1 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: kSnappyCompression supported: 1 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-16T23:52:27.923 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi194/store.db/MANIFEST-000003 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.merge_operator: 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_filter: None 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_filter_factory: None 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.sst_partitioner_factory: None 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-16T23:52:27.924 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5572dd8c3d00) 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: cache_index_and_filter_blocks: 1 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: pin_top_level_index_and_filter: 1 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_type: 0 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_block_index_type: 0 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_shortening: 1 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-16T23:52:27.925 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: hash_index_allow_collision: 1 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: checksum: 1 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: no_block_cache: 0 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache: 0x5572dd92e170 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_name: BinnedLRUCache 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_options: 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: capacity : 536870912 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: num_shard_bits : 4 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: strict_capacity_limit : 0 2024-09-16T23:52:27.926 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: high_pri_pool_ratio: 0.000 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_compressed: (nil) 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: persistent_cache: (nil) 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_size: 4096 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_size_deviation: 10 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_restart_interval: 16 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_block_restart_interval: 1 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: metadata_block_size: 4096 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: partition_filters: 0 2024-09-16T23:52:27.927 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: use_delta_encoding: 1 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: whole_key_filtering: 1 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: verify_compression: 0 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: read_amp_bytes_per_bit: 0 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: format_version: 4 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: enable_index_compression: 1 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_align: 0 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.write_buffer_size: 33554432 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_write_buffer_number: 2 2024-09-16T23:52:27.928 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression: NoCompression 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression: Disabled 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.prefix_extractor: nullptr 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.num_levels: 7 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-16T23:52:27.929 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.level: 32767 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.strategy: 0 2024-09-16T23:52:27.930 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compression_opts.enabled: false 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.target_file_size_base: 67108864 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-16T23:52:27.931 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-16T23:52:27.932 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.arena_block_size: 4194304 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.disable_auto_compactions: 0 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-16T23:52:27.933 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.table_properties_collectors: 2024-09-16T23:52:27.934 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.inplace_update_support: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.bloom_locality: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.max_successive_merges: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.paranoid_file_checks: 0 2024-09-16T23:52:27.935 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.force_consistency_checks: 1 2024-09-16T23:52:27.936 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.report_bg_io_stats: 0 2024-09-16T23:52:27.936 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.ttl: 2592000 2024-09-16T23:52:27.936 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-16T23:52:27.936 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.enable_blob_files: false 2024-09-16T23:52:27.936 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.min_blob_size: 0 2024-09-16T23:52:27.936 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.blob_file_size: 268435456 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi194/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726530747663258, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-16T23:52:27.937 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-16T23:52:27.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-16T23:52:27.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726530747663568, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726530747, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "02524b4f-e1ea-454d-b577-e3a2e61ad573", "db_session_id": "L9TN3VBEDRLLUWBMOEQB"}} 2024-09-16T23:52:27.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-16T23:52:27.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726530747663884, "job": 1, "event": "recovery_finished"} 2024-09-16T23:52:27.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi194/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-16T23:52:27.938 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x5572dd914a80 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: DB pointer 0x5572dd924000 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** DB Stats ** 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T23:52:27.939 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.940 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T23:52:27.941 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.942 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-16T23:52:27.943 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-16T23:52:27.944 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194 does not exist in monmap, will attempt to join an existing cluster 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: using public_addr v2:172.21.15.194:0/0 -> [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: starting mon.smithi194 rank -1 at public addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] at bind addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi194 fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(???) e0 preinit fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).mds e1 new map 2024-09-16T23:52:27.945 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).mds e1 print_map 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: e1 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: legacy client fscid: -1 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: No filesystems configured 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-16T23:52:27.946 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Deploying daemon crash.smithi007 on smithi007 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.947 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Deploying daemon grafana.smithi007 on smithi007 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/267615095' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/267615095' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: osdmap e4: 0 total, 0 up, 0 in 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.948 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi007:172.21.15.7=smithi007;smithi194:172.21.15.194=smithi194", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Saving service mon spec with placement smithi007:172.21.15.7=smithi007;smithi194:172.21.15.194=smithi194;count:2 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/119696515' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Deploying daemon node-exporter.smithi007 on smithi007 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/2338446615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.949 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14162 172.21.15.7:0/1399907936' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mgrmap e14: smithi007.czdphd(active, since 42s) 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/3405728631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/1916497105' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Active manager daemon smithi007.czdphd restarted 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Activating manager daemon smithi007.czdphd 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mgrmap e15: smithi007.czdphd(active, starting, since 0.0523531s) 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-16T23:52:27.950 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Manager daemon smithi007.czdphd is now available 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/321670129' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.951 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: [16/Sep/2024:23:52:09] ENGINE Bus STARTING 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: [16/Sep/2024:23:52:09] ENGINE Serving on https://172.21.15.7:7150 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: [16/Sep/2024:23:52:09] ENGINE Bus STARTED 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mgrmap e16: smithi007.czdphd(active, since 1.05476s) 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mgrmap e17: smithi007.czdphd(active, since 2s) 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi007", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:52:27.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/2958478254' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd/host:smithi194", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Deploying daemon crash.smithi194 on smithi194 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:27.953 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/1324014835' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Deploying daemon mgr.smithi194.xlteft on smithi194 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/2965647238' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: Deploying daemon mon.smithi194 on smithi194 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/250922313' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 ceph-mon[25713]: mon.smithi194@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-16T23:52:27.954 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 bash[25670]: 32c4fa431c9c102b08ccac3d9ff3a24e0fa1ada9ec0e0a9f7e08cdf7da2d5171 2024-09-16T23:52:27.955 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:27 smithi194 systemd[1]: Started Ceph mon.smithi194 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-16T23:52:28.536 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-16T23:52:28.536 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-16T23:52:32.984 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:52:32.984 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: mon.smithi007 calling monitor election 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: mon.smithi194 calling monitor election 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: mon.smithi007 is new leader, mons smithi007,smithi194 in quorum (ranks 0,1) 2024-09-16T23:52:32.985 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: monmap e2: 2 mons at {smithi007=[v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0],smithi194=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0]} 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: fsmap 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: mgrmap e17: smithi007.czdphd(active, since 23s) 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft started 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: overall HEALTH_OK 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:32.986 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T23:52:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:52:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: mon.smithi007 calling monitor election 2024-09-16T23:52:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-16T23:52:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: mon.smithi194 calling monitor election 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: mon.smithi007 is new leader, mons smithi007,smithi194 in quorum (ranks 0,1) 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: monmap e2: 2 mons at {smithi007=[v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0],smithi194=[v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0]} 2024-09-16T23:52:33.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: fsmap 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: osdmap e5: 0 total, 0 up, 0 in 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: mgrmap e17: smithi007.czdphd(active, since 23s) 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: Standby manager daemon smithi194.xlteft started 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: overall HEALTH_OK 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:33.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/2901045459' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-16T23:52:33.340 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:52:33.340 INFO:teuthology.orchestra.run.smithi194.stdout:{"epoch":2,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","modified":"2024-09-16T23:52:27.691419Z","created":"2024-09-16T23:50:21.589767Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi007","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:3300","nonce":0},{"type":"v1","addr":"172.21.15.7:6789","nonce":0}]},"addr":"172.21.15.7:6789/0","public_addr":"172.21.15.7:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi194","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:3300","nonce":0},{"type":"v1","addr":"172.21.15.194:6789","nonce":0}]},"addr":"172.21.15.194:6789/0","public_addr":"172.21.15.194:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-16T23:52:33.342 INFO:teuthology.orchestra.run.smithi194.stderr:dumped monmap epoch 2 2024-09-16T23:52:33.900 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:33 smithi194 ceph-mon[25713]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-16T23:52:33.900 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:33 smithi194 ceph-mon[25713]: mgrmap e18: smithi007.czdphd(active, since 23s), standbys: smithi194.xlteft 2024-09-16T23:52:33.900 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:33 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-16T23:52:33.900 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:33 smithi194 ceph-mon[25713]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:33.900 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:33 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/88243075' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:33.901 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:33 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:33.922 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-16T23:52:33.922 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-16T23:52:34.012 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:33 smithi007 ceph-mon[23721]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-16T23:52:34.012 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:33 smithi007 ceph-mon[23721]: mgrmap e18: smithi007.czdphd(active, since 23s), standbys: smithi194.xlteft 2024-09-16T23:52:34.012 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:33 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-16T23:52:34.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:33 smithi007 ceph-mon[23721]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:34.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:33 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/88243075' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-16T23:52:34.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:33 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:52:35.338 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:35 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:35.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:35 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:35.832 INFO:teuthology.orchestra.run.smithi007.stdout:# minimal ceph.conf for 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:52:35.832 INFO:teuthology.orchestra.run.smithi007.stdout:[global] 2024-09-16T23:52:35.832 INFO:teuthology.orchestra.run.smithi007.stdout: fsid = 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-16T23:52:35.832 INFO:teuthology.orchestra.run.smithi007.stdout: mon_host = [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] 2024-09-16T23:52:36.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:36 smithi194 ceph-mon[25713]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:36.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:36 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1265087594' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:36.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:36 smithi007 ceph-mon[23721]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:36.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:36 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1265087594' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:36.454 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-16T23:52:36.455 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:52:36.455 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd of=/etc/ceph/ceph.conf 2024-09-16T23:52:36.486 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:52:36.486 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:52:36.554 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-16T23:52:36.554 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.conf 2024-09-16T23:52:36.585 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-16T23:52:36.585 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:52:36.652 INFO:tasks.cephadm:Deploying OSDs... 2024-09-16T23:52:36.652 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:52:36.652 DEBUG:teuthology.orchestra.run.smithi007:> dd if=/scratch_devs of=/dev/stdout 2024-09-16T23:52:36.670 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-16T23:52:36.670 DEBUG:teuthology.orchestra.run.smithi007:> stat /dev/vg_nvme/lv_1 2024-09-16T23:52:36.726 INFO:teuthology.orchestra.run.smithi007.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-16T23:52:36.726 INFO:teuthology.orchestra.run.smithi007.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:36.726 INFO:teuthology.orchestra.run.smithi007.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-16T23:52:36.727 INFO:teuthology.orchestra.run.smithi007.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:36.727 INFO:teuthology.orchestra.run.smithi007.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:36.727 INFO:teuthology.orchestra.run.smithi007.stdout:Access: 2024-09-16 23:51:59.299935801 +0000 2024-09-16T23:52:36.727 INFO:teuthology.orchestra.run.smithi007.stdout:Modify: 2024-09-16 23:49:30.122487916 +0000 2024-09-16T23:52:36.727 INFO:teuthology.orchestra.run.smithi007.stdout:Change: 2024-09-16 23:49:30.122487916 +0000 2024-09-16T23:52:36.727 INFO:teuthology.orchestra.run.smithi007.stdout: Birth: 2024-09-16 23:49:30.122487916 +0000 2024-09-16T23:52:36.727 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-16T23:52:36.791 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records in 2024-09-16T23:52:36.791 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records out 2024-09-16T23:52:36.791 INFO:teuthology.orchestra.run.smithi007.stderr:512 bytes copied, 0.000108403 s, 4.7 MB/s 2024-09-16T23:52:36.793 DEBUG:teuthology.orchestra.run.smithi007:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-16T23:52:36.849 DEBUG:teuthology.orchestra.run.smithi007:> stat /dev/vg_nvme/lv_2 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout:Access: 2024-09-16 23:51:59.299935801 +0000 2024-09-16T23:52:36.907 INFO:teuthology.orchestra.run.smithi007.stdout:Modify: 2024-09-16 23:49:30.442483271 +0000 2024-09-16T23:52:36.908 INFO:teuthology.orchestra.run.smithi007.stdout:Change: 2024-09-16 23:49:30.442483271 +0000 2024-09-16T23:52:36.908 INFO:teuthology.orchestra.run.smithi007.stdout: Birth: 2024-09-16 23:49:30.442483271 +0000 2024-09-16T23:52:36.908 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-16T23:52:36.974 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records in 2024-09-16T23:52:36.974 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records out 2024-09-16T23:52:36.974 INFO:teuthology.orchestra.run.smithi007.stderr:512 bytes copied, 0.000118904 s, 4.3 MB/s 2024-09-16T23:52:36.976 DEBUG:teuthology.orchestra.run.smithi007:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-16T23:52:37.032 DEBUG:teuthology.orchestra.run.smithi007:> stat /dev/vg_nvme/lv_3 2024-09-16T23:52:37.088 INFO:teuthology.orchestra.run.smithi007.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-16T23:52:37.088 INFO:teuthology.orchestra.run.smithi007.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout:Access: 2024-09-16 23:51:59.300935786 +0000 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout:Modify: 2024-09-16 23:49:30.766478569 +0000 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout:Change: 2024-09-16 23:49:30.766478569 +0000 2024-09-16T23:52:37.089 INFO:teuthology.orchestra.run.smithi007.stdout: Birth: 2024-09-16 23:49:30.766478569 +0000 2024-09-16T23:52:37.089 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-16T23:52:37.153 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records in 2024-09-16T23:52:37.153 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records out 2024-09-16T23:52:37.153 INFO:teuthology.orchestra.run.smithi007.stderr:512 bytes copied, 0.000110905 s, 4.6 MB/s 2024-09-16T23:52:37.154 DEBUG:teuthology.orchestra.run.smithi007:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-16T23:52:37.212 DEBUG:teuthology.orchestra.run.smithi007:> stat /dev/vg_nvme/lv_4 2024-09-16T23:52:37.268 INFO:teuthology.orchestra.run.smithi007.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-16T23:52:37.268 INFO:teuthology.orchestra.run.smithi007.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:37.268 INFO:teuthology.orchestra.run.smithi007.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-09-16T23:52:37.269 INFO:teuthology.orchestra.run.smithi007.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:37.269 INFO:teuthology.orchestra.run.smithi007.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:37.269 INFO:teuthology.orchestra.run.smithi007.stdout:Access: 2024-09-16 23:51:59.300935786 +0000 2024-09-16T23:52:37.269 INFO:teuthology.orchestra.run.smithi007.stdout:Modify: 2024-09-16 23:49:31.092473837 +0000 2024-09-16T23:52:37.269 INFO:teuthology.orchestra.run.smithi007.stdout:Change: 2024-09-16 23:49:31.092473837 +0000 2024-09-16T23:52:37.269 INFO:teuthology.orchestra.run.smithi007.stdout: Birth: 2024-09-16 23:49:31.092473837 +0000 2024-09-16T23:52:37.269 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-16T23:52:37.333 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records in 2024-09-16T23:52:37.333 INFO:teuthology.orchestra.run.smithi007.stderr:1+0 records out 2024-09-16T23:52:37.333 INFO:teuthology.orchestra.run.smithi007.stderr:512 bytes copied, 0.000109187 s, 4.7 MB/s 2024-09-16T23:52:37.334 DEBUG:teuthology.orchestra.run.smithi007:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-16T23:52:37.391 DEBUG:teuthology.orchestra.run.smithi194:> set -ex 2024-09-16T23:52:37.391 DEBUG:teuthology.orchestra.run.smithi194:> dd if=/scratch_devs of=/dev/stdout 2024-09-16T23:52:37.410 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-16T23:52:37.410 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_1 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-16 23:52:33.282906902 +0000 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-16 23:49:26.338103017 +0000 2024-09-16T23:52:37.469 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-16 23:49:26.338103017 +0000 2024-09-16T23:52:37.470 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-16 23:49:26.338103017 +0000 2024-09-16T23:52:37.470 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-16T23:52:37.535 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-16T23:52:37.535 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-16T23:52:37.535 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000232917 s, 2.2 MB/s 2024-09-16T23:52:37.536 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-16T23:52:37.595 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_2 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-16 23:52:33.282906902 +0000 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-16 23:49:26.668095934 +0000 2024-09-16T23:52:37.651 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-16 23:49:26.668095934 +0000 2024-09-16T23:52:37.652 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-16 23:49:26.668095934 +0000 2024-09-16T23:52:37.652 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-16T23:52:37.716 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-16T23:52:37.717 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-16T23:52:37.717 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000241524 s, 2.1 MB/s 2024-09-16T23:52:37.718 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-16T23:52:37.775 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_3 2024-09-16T23:52:37.831 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 708 Links: 1 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-16 23:52:33.283906881 +0000 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-16 23:49:26.989089046 +0000 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-16 23:49:26.989089046 +0000 2024-09-16T23:52:37.832 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-16 23:49:26.989089046 +0000 2024-09-16T23:52:37.832 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-16T23:52:37.893 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:37.893 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:37 smithi194 ceph-mon[25713]: Deploying daemon prometheus.smithi007 on smithi007 2024-09-16T23:52:37.894 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:37 smithi194 ceph-mon[25713]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:37.896 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-16T23:52:37.896 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-16T23:52:37.896 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000240741 s, 2.1 MB/s 2024-09-16T23:52:37.897 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-16T23:52:37.954 DEBUG:teuthology.orchestra.run.smithi194:> stat /dev/vg_nvme/lv_4 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout:Context: system_u:object_r:device_t:s0 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout:Access: 2024-09-16 23:52:33.283906881 +0000 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout:Modify: 2024-09-16 23:49:27.321081921 +0000 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout:Change: 2024-09-16 23:49:27.321081921 +0000 2024-09-16T23:52:38.010 INFO:teuthology.orchestra.run.smithi194.stdout: Birth: 2024-09-16 23:49:27.321081921 +0000 2024-09-16T23:52:38.011 DEBUG:teuthology.orchestra.run.smithi194:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-16T23:52:38.073 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records in 2024-09-16T23:52:38.074 INFO:teuthology.orchestra.run.smithi194.stderr:1+0 records out 2024-09-16T23:52:38.074 INFO:teuthology.orchestra.run.smithi194.stderr:512 bytes copied, 0.000200643 s, 2.6 MB/s 2024-09-16T23:52:38.075 DEBUG:teuthology.orchestra.run.smithi194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-16T23:52:38.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:38.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:37 smithi007 ceph-mon[23721]: Deploying daemon prometheus.smithi007 on smithi007 2024-09-16T23:52:38.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:37 smithi007 ceph-mon[23721]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:38.132 INFO:tasks.cephadm:Deploying osd.0 on smithi007 with /dev/vg_nvme/lv_4... 2024-09-16T23:52:38.132 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-16T23:52:40.348 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:40 smithi007 ceph-mon[23721]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:40.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:40 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:40 smithi194 ceph-mon[25713]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:40 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:42.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:42 smithi007 ceph-mon[23721]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:42.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:42 smithi194 ceph-mon[25713]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:44.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:44 smithi007 ceph-mon[23721]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:44 smithi194 ceph-mon[25713]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:46.078 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:45 smithi007 ceph-mon[23721]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:46.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:45 smithi194 ceph-mon[25713]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:48.212 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:48 smithi007 ceph-mon[23721]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:48.212 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:48.212 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:52:48.212 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:48.212 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:52:48.364 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:48 smithi194 ceph-mon[25713]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:48.364 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:48.364 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:52:48.364 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:48.364 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:52:48.925 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:52:48.947 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi007:vg_nvme/lv_4 2024-09-16T23:52:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:49 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:49 smithi007 ceph-mon[23721]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-16T23:52:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:49 smithi007 ceph-mon[23721]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:52:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:49 smithi007 ceph-mon[23721]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:49 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:49 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:49 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:49 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-16T23:52:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:49 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-16T23:52:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:49 smithi194 ceph-mon[25713]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:49 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:49 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:52.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:52 smithi007 ceph-mon[23721]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:52.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:52:52.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:52:52.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:52.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:52 smithi194 ceph-mon[25713]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:52.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:52:52.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:52:52.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:53.347 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:53 smithi007 ceph-mon[23721]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:52:53.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:53 smithi194 ceph-mon[25713]: from='client.14234 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: Reconfiguring mon.smithi007 (unknown last config time)... 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:54.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:54 smithi007 ceph-mon[23721]: Reconfiguring daemon mon.smithi007 on smithi007 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: Reconfiguring mon.smithi007 (unknown last config time)... 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:54 smithi194 ceph-mon[25713]: Reconfiguring daemon mon.smithi007 on smithi007 2024-09-16T23:52:56.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:56.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:56.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: Reconfiguring mgr.smithi007.czdphd (unknown last config time)... 2024-09-16T23:52:56.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:52:56.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:52:56.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:56.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:56 smithi007 ceph-mon[23721]: Reconfiguring daemon mgr.smithi007.czdphd on smithi007 2024-09-16T23:52:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:52:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: Reconfiguring mgr.smithi007.czdphd (unknown last config time)... 2024-09-16T23:52:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:52:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:52:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:52:56.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:56 smithi194 ceph-mon[25713]: Reconfiguring daemon mgr.smithi007.czdphd on smithi007 2024-09-16T23:52:57.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:57 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/425984978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b298b2c2-3864-42e7-bf52-1ffbf6adb517"}]: dispatch 2024-09-16T23:52:57.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:57 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/425984978' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b298b2c2-3864-42e7-bf52-1ffbf6adb517"}]': finished 2024-09-16T23:52:57.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:57 smithi007 ceph-mon[23721]: osdmap e6: 1 total, 0 up, 1 in 2024-09-16T23:52:57.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:57 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:52:57.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:57 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1237120356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:52:57.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:57 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/425984978' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b298b2c2-3864-42e7-bf52-1ffbf6adb517"}]: dispatch 2024-09-16T23:52:57.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:57 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/425984978' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b298b2c2-3864-42e7-bf52-1ffbf6adb517"}]': finished 2024-09-16T23:52:57.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:57 smithi194 ceph-mon[25713]: osdmap e6: 1 total, 0 up, 1 in 2024-09-16T23:52:57.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:57 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:52:57.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:57 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1237120356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:52:58.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:52:58 smithi007 ceph-mon[23721]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:52:58.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:52:58 smithi194 ceph-mon[25713]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:00.558 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:00 smithi007 ceph-mon[23721]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:00.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:00 smithi194 ceph-mon[25713]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:01.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:01.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:01 smithi007 ceph-mon[23721]: Reconfiguring alertmanager.smithi007 (dependencies changed)... 2024-09-16T23:53:01.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:01 smithi007 ceph-mon[23721]: Reconfiguring daemon alertmanager.smithi007 on smithi007 2024-09-16T23:53:01.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:01 smithi007 ceph-mon[23721]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:01.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:01.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:01 smithi194 ceph-mon[25713]: Reconfiguring alertmanager.smithi007 (dependencies changed)... 2024-09-16T23:53:01.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:01 smithi194 ceph-mon[25713]: Reconfiguring daemon alertmanager.smithi007 on smithi007 2024-09-16T23:53:01.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:01 smithi194 ceph-mon[25713]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:04.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:04 smithi007 ceph-mon[23721]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:04.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:04 smithi194 ceph-mon[25713]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:06.056 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:05 smithi007 ceph-mon[23721]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:06.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:05 smithi194 ceph-mon[25713]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:07 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:07 smithi007 ceph-mon[23721]: Reconfiguring crash.smithi007 (monmap changed)... 2024-09-16T23:53:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:07 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:53:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:07 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:07 smithi007 ceph-mon[23721]: Reconfiguring daemon crash.smithi007 on smithi007 2024-09-16T23:53:07.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:07 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:07.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:07 smithi194 ceph-mon[25713]: Reconfiguring crash.smithi007 (monmap changed)... 2024-09-16T23:53:07.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:07 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:53:07.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:07 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:07.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:07 smithi194 ceph-mon[25713]: Reconfiguring daemon crash.smithi007 on smithi007 2024-09-16T23:53:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:08 smithi007 ceph-mon[23721]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-16T23:53:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:08 smithi194 ceph-mon[25713]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-16T23:53:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:09.326 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:09 smithi007 ceph-mon[23721]: Deploying daemon osd.0 on smithi007 2024-09-16T23:53:09.327 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:53:09.327 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:53:09.327 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:09 smithi194 ceph-mon[25713]: Deploying daemon osd.0 on smithi007 2024-09-16T23:53:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:53:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:53:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:10.467 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:10 smithi007 ceph-mon[23721]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:10.467 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:10 smithi007 ceph-mon[23721]: Reconfiguring grafana.smithi007 (dependencies changed)... 2024-09-16T23:53:10.467 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:10 smithi007 ceph-mon[23721]: Reconfiguring daemon grafana.smithi007 on smithi007 2024-09-16T23:53:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:10 smithi194 ceph-mon[25713]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:10 smithi194 ceph-mon[25713]: Reconfiguring grafana.smithi007 (dependencies changed)... 2024-09-16T23:53:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:10 smithi194 ceph-mon[25713]: Reconfiguring daemon grafana.smithi007 on smithi007 2024-09-16T23:53:12.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:12 smithi007 ceph-mon[23721]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:12 smithi194 ceph-mon[25713]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:14.413 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:14 smithi007 ceph-mon[23721]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:14.413 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:14 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:14.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:14 smithi194 ceph-mon[25713]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:14.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:14 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:15.959 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:15 smithi007 ceph-mon[23721]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:15 smithi194 ceph-mon[25713]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:18.247 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:18 smithi007 ceph-mon[23721]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:18.248 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:18 smithi007 ceph-mon[23721]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-16T23:53:18.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:18 smithi194 ceph-mon[25713]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:18.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:18 smithi194 ceph-mon[25713]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-16T23:53:19.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:19 smithi007 ceph-mon[23721]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T23:53:19.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:19 smithi007 ceph-mon[23721]: osdmap e7: 1 total, 0 up, 1 in 2024-09-16T23:53:19.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:19.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:19 smithi007 ceph-mon[23721]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:53:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:19 smithi194 ceph-mon[25713]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-16T23:53:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:19 smithi194 ceph-mon[25713]: osdmap e7: 1 total, 0 up, 1 in 2024-09-16T23:53:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:19 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:19 smithi194 ceph-mon[25713]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: osdmap e8: 1 total, 0 up, 1 in 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:20.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:53:20.136 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: from='osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: osdmap e8: 1 total, 0 up, 1 in 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-16T23:53:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667] boot 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: osdmap e9: 1 total, 1 up, 1 in 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:21.154 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:53:21.155 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:53:21.155 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:21.162 INFO:teuthology.orchestra.run.smithi007.stdout:Created osd(s) 0 on host 'smithi007' 2024-09-16T23:53:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:53:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:53:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-16T23:53:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: osd.0 [v2:172.21.15.7:6802/3763790667,v1:172.21.15.7:6803/3763790667] boot 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: osdmap e9: 1 total, 1 up, 1 in 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:53:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:21.752 DEBUG:teuthology.orchestra.run.smithi007:osd.0> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.0.service 2024-09-16T23:53:21.754 INFO:tasks.cephadm:Deploying osd.1 on smithi007 with /dev/vg_nvme/lv_3... 2024-09-16T23:53:21.755 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-16T23:53:22.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:22 smithi007 ceph-mon[23721]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:22.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:22 smithi007 ceph-mon[23721]: Reconfiguring mgr.smithi194.xlteft (monmap changed)... 2024-09-16T23:53:22.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:22 smithi007 ceph-mon[23721]: Reconfiguring daemon mgr.smithi194.xlteft on smithi194 2024-09-16T23:53:22.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:22 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:22.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:22 smithi007 ceph-mon[23721]: osdmap e10: 1 total, 1 up, 1 in 2024-09-16T23:53:22.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:22 smithi194 ceph-mon[25713]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-16T23:53:22.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:22 smithi194 ceph-mon[25713]: Reconfiguring mgr.smithi194.xlteft (monmap changed)... 2024-09-16T23:53:22.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:22 smithi194 ceph-mon[25713]: Reconfiguring daemon mgr.smithi194.xlteft on smithi194 2024-09-16T23:53:22.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:22 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:22.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:22 smithi194 ceph-mon[25713]: osdmap e10: 1 total, 1 up, 1 in 2024-09-16T23:53:23.289 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:23.290 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:23 smithi007 ceph-mon[23721]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-16T23:53:23.290 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T23:53:23.290 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T23:53:23.290 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:23 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:23.290 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:23 smithi007 ceph-mon[23721]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-16T23:53:23.484 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:23 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:23.484 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:23 smithi194 ceph-mon[25713]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-16T23:53:23.485 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:23 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-16T23:53:23.485 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:23 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-16T23:53:23.485 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:23 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:23.485 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:23 smithi194 ceph-mon[25713]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-16T23:53:24.443 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:24.443 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.443 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T23:53:24.443 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T23:53:24.443 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.7:9093"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.7:9093"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.7:3000"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.7:3000"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.7:9095"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.7:9095"}]: dispatch 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.444 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:53:24.445 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:24.445 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:53:24.740 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:53:24.758 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: pgmap v35: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:24.758 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.758 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.7:9093"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.7:9093"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.7:3000"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.7:3000"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.7:9095"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.7:9095"}]: dispatch 2024-09-16T23:53:24.759 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:24.760 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:53:24.760 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:24.760 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:53:24.761 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi007:vg_nvme/lv_3 2024-09-16T23:53:26.340 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:26 smithi007 ceph-mon[23721]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:26.340 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:26 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:26.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:26 smithi194 ceph-mon[25713]: pgmap v36: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:26 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:28 smithi007 ceph-mon[23721]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:28 smithi007 ceph-mon[23721]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:53:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:28 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:53:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:28 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:53:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:28 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:28 smithi194 ceph-mon[25713]: pgmap v37: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:28 smithi194 ceph-mon[25713]: from='client.14252 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:53:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:28 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:53:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:28 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:53:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:28 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:30.339 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:30 smithi007 ceph-mon[23721]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:30.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:30 smithi194 ceph-mon[25713]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:31.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:31 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:31.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:31 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2149899281' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e2a2d65a-c40b-45ba-b2a9-051d7301c685"}]: dispatch 2024-09-16T23:53:31.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:31 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2149899281' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e2a2d65a-c40b-45ba-b2a9-051d7301c685"}]': finished 2024-09-16T23:53:31.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:31 smithi007 ceph-mon[23721]: osdmap e11: 2 total, 1 up, 2 in 2024-09-16T23:53:31.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:31 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:31.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:31 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:31.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:31 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2149899281' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e2a2d65a-c40b-45ba-b2a9-051d7301c685"}]: dispatch 2024-09-16T23:53:31.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:31 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2149899281' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e2a2d65a-c40b-45ba-b2a9-051d7301c685"}]': finished 2024-09-16T23:53:31.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:31 smithi194 ceph-mon[25713]: osdmap e11: 2 total, 1 up, 2 in 2024-09-16T23:53:31.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:31 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:32.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:32 smithi007 ceph-mon[23721]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:32.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:32 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/601709102' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:53:32.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:32 smithi194 ceph-mon[25713]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:32 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/601709102' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:53:34.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:34 smithi007 ceph-mon[23721]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:34 smithi194 ceph-mon[25713]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:36.033 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:35 smithi007 ceph-mon[23721]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:36.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:35 smithi194 ceph-mon[25713]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi007 2024-09-16T23:53:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:53:38.401 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi007 to 12868M 2024-09-16T23:53:38.402 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:38.402 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:38.402 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:53:38.402 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:38.402 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi007 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi007 to 12868M 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:38.552 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:53:39.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:39 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-16T23:53:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:39 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:39 smithi194 ceph-mon[25713]: Deploying daemon osd.1 on smithi007 2024-09-16T23:53:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:39 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:39.500 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:39 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-16T23:53:39.500 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:39 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:39.500 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:39 smithi007 ceph-mon[23721]: Deploying daemon osd.1 on smithi007 2024-09-16T23:53:39.501 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:39 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:40 smithi194 ceph-mon[25713]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:40.464 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:40 smithi007 ceph-mon[23721]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:42.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:42 smithi007 ceph-mon[23721]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:42.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:42 smithi194 ceph-mon[25713]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:44 smithi194 ceph-mon[25713]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:44.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:44 smithi007 ceph-mon[23721]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:45.557 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:45.558 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:46.429 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:46 smithi007 ceph-mon[23721]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:46.429 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:46.429 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:53:46.429 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:46.429 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:53:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:46 smithi194 ceph-mon[25713]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:53:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:53:48.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:48 smithi007 ceph-mon[23721]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:48.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:48.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:48 smithi194 ceph-mon[25713]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:48.973 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:49.424 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:49 smithi007 ceph-mon[23721]: from='osd.1 [v2:172.21.15.7:6810/3862868244,v1:172.21.15.7:6811/3862868244]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T23:53:49.424 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:49 smithi007 ceph-mon[23721]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T23:53:49.853 INFO:teuthology.orchestra.run.smithi007.stdout:Created osd(s) 1 on host 'smithi007' 2024-09-16T23:53:49.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:49 smithi194 ceph-mon[25713]: from='osd.1 [v2:172.21.15.7:6810/3862868244,v1:172.21.15.7:6811/3862868244]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T23:53:49.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:49 smithi194 ceph-mon[25713]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-16T23:53:50.449 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:50.449 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T23:53:50.449 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: osdmap e12: 2 total, 1 up, 2 in 2024-09-16T23:53:50.450 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:50.450 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: from='osd.1 [v2:172.21.15.7:6810/3862868244,v1:172.21.15.7:6811/3862868244]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:53:50.450 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:53:50.450 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:50.451 DEBUG:teuthology.orchestra.run.smithi007:osd.1> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.1.service 2024-09-16T23:53:50.453 INFO:tasks.cephadm:Deploying osd.2 on smithi007 with /dev/vg_nvme/lv_2... 2024-09-16T23:53:50.454 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: osdmap e12: 2 total, 1 up, 2 in 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: from='osd.1 [v2:172.21.15.7:6810/3862868244,v1:172.21.15.7:6811/3862868244]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:53:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:51.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:51 smithi007 ceph-mon[23721]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:53:51.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:51 smithi007 ceph-mon[23721]: osdmap e13: 2 total, 1 up, 2 in 2024-09-16T23:53:51.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:51 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:51.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:51 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:51.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:51 smithi194 ceph-mon[25713]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:53:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:51 smithi194 ceph-mon[25713]: osdmap e13: 2 total, 1 up, 2 in 2024-09-16T23:53:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:51 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:51 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:52.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:53:52.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:53:52.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:52.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:52.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: osd.1 [v2:172.21.15.7:6810/3862868244,v1:172.21.15.7:6811/3862868244] boot 2024-09-16T23:53:52.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: osdmap e14: 2 total, 2 up, 2 in 2024-09-16T23:53:52.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: osd.1 [v2:172.21.15.7:6810/3862868244,v1:172.21.15.7:6811/3862868244] boot 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: osdmap e14: 2 total, 2 up, 2 in 2024-09-16T23:53:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-16T23:53:53.611 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:53 smithi007 ceph-mon[23721]: osdmap e15: 2 total, 2 up, 2 in 2024-09-16T23:53:53.616 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:53:53.639 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi007:vg_nvme/lv_2 2024-09-16T23:53:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:53 smithi194 ceph-mon[25713]: osdmap e15: 2 total, 2 up, 2 in 2024-09-16T23:53:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:54 smithi007 ceph-mon[23721]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:53:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:54 smithi194 ceph-mon[25713]: pgmap v55: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:53:55.801 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:55 smithi007 ceph-mon[23721]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:53:56.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:55 smithi194 ceph-mon[25713]: pgmap v56: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:53:57.089 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:57.090 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:56 smithi007 ceph-mon[23721]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:53:57.090 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:53:57.090 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:53:57.090 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:56 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:57.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:53:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:56 smithi194 ceph-mon[25713]: from='client.14270 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:53:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:53:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:53:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:56 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:53:57.964 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:53:57 smithi007 ceph-mon[23721]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:53:58.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:53:57 smithi194 ceph-mon[25713]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:00 smithi007 ceph-mon[23721]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:00.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:00 smithi194 ceph-mon[25713]: pgmap v58: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi007 to 6434M 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:01.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi007 to 6434M 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:02 smithi194 ceph-mon[25713]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:02 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/539844712' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7225a34e-aff2-4991-a631-d895e13a92ce"}]: dispatch 2024-09-16T23:54:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:02 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/539844712' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7225a34e-aff2-4991-a631-d895e13a92ce"}]': finished 2024-09-16T23:54:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:02 smithi194 ceph-mon[25713]: osdmap e16: 3 total, 2 up, 3 in 2024-09-16T23:54:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:02 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:02 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3398041520' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:54:02.418 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:02 smithi007 ceph-mon[23721]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:02.418 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:02 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/539844712' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7225a34e-aff2-4991-a631-d895e13a92ce"}]: dispatch 2024-09-16T23:54:02.418 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:02 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/539844712' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7225a34e-aff2-4991-a631-d895e13a92ce"}]': finished 2024-09-16T23:54:02.418 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:02 smithi007 ceph-mon[23721]: osdmap e16: 3 total, 2 up, 3 in 2024-09-16T23:54:02.418 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:02 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:02.418 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:02 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3398041520' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:54:04.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:04 smithi007 ceph-mon[23721]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:04.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:04 smithi194 ceph-mon[25713]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:06.268 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:06 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:06.269 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:06 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:06.269 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:06 smithi007 ceph-mon[23721]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:06.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:06 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:06.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:06 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:06.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:06 smithi194 ceph-mon[25713]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:08.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:08 smithi007 ceph-mon[23721]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:08.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-16T23:54:08.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:08.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:08 smithi007 ceph-mon[23721]: Deploying daemon osd.2 on smithi007 2024-09-16T23:54:08.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:08 smithi194 ceph-mon[25713]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-16T23:54:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:08 smithi194 ceph-mon[25713]: Deploying daemon osd.2 on smithi007 2024-09-16T23:54:09.285 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:54:09.285 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:54:09.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:54:09.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:54:10.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:10 smithi194 ceph-mon[25713]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:10.508 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:10 smithi007 ceph-mon[23721]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:12.229 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:12 smithi007 ceph-mon[23721]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:12.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:12 smithi194 ceph-mon[25713]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:13 smithi007 ceph-mon[23721]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:14.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:14.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:14.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:14.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:14.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:13 smithi194 ceph-mon[25713]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:15 smithi007 ceph-mon[23721]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:15 smithi194 ceph-mon[25713]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:16.527 INFO:teuthology.orchestra.run.smithi007.stdout:Created osd(s) 2 on host 'smithi007' 2024-09-16T23:54:17.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:16 smithi007 ceph-mon[23721]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-16T23:54:17.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:16 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:17.093 DEBUG:teuthology.orchestra.run.smithi007:osd.2> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.2.service 2024-09-16T23:54:17.096 INFO:tasks.cephadm:Deploying osd.3 on smithi007 with /dev/vg_nvme/lv_1... 2024-09-16T23:54:17.096 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-16T23:54:17.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:16 smithi194 ceph-mon[25713]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-16T23:54:17.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:16 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-mon[23721]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T23:54:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-mon[23721]: osdmap e17: 3 total, 2 up, 3 in 2024-09-16T23:54:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-mon[23721]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:54:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-mon[23721]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:18.100 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[41865]: 2024-09-16T23:54:17.798+0000 7f69d05d0700 -1 osd.2 0 waiting for initial osdmap 2024-09-16T23:54:18.100 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:17 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[41865]: 2024-09-16T23:54:17.802+0000 7f69c8f63700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T23:54:18.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:17 smithi194 ceph-mon[25713]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-16T23:54:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:17 smithi194 ceph-mon[25713]: osdmap e17: 3 total, 2 up, 3 in 2024-09-16T23:54:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:17 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:17 smithi194 ceph-mon[25713]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:54:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:17 smithi194 ceph-mon[25713]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:19.079 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:18 smithi007 ceph-mon[23721]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:54:19.079 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:18 smithi007 ceph-mon[23721]: osdmap e18: 3 total, 2 up, 3 in 2024-09-16T23:54:19.079 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:18 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:19.079 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:18 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:19.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:18 smithi194 ceph-mon[25713]: from='osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:54:19.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:18 smithi194 ceph-mon[25713]: osdmap e18: 3 total, 2 up, 3 in 2024-09-16T23:54:19.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:18 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:19.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:18 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:54:20.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:54:20.109 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106] boot 2024-09-16T23:54:20.109 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: osdmap e19: 3 total, 3 up, 3 in 2024-09-16T23:54:20.109 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:20.109 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:20.109 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:19 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: osd.2 [v2:172.21.15.7:6818/3183589106,v1:172.21.15.7:6819/3183589106] boot 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: osdmap e19: 3 total, 3 up, 3 in 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: pgmap v72: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2024-09-16T23:54:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:19 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-16T23:54:20.506 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:54:20.534 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi007:vg_nvme/lv_1 2024-09-16T23:54:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-16T23:54:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:20 smithi007 ceph-mon[23721]: osdmap e20: 3 total, 3 up, 3 in 2024-09-16T23:54:21.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-16T23:54:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-16T23:54:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:20 smithi194 ceph-mon[25713]: osdmap e20: 3 total, 3 up, 3 in 2024-09-16T23:54:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-16T23:54:22.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-16T23:54:22.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:21 smithi007 ceph-mon[23721]: osdmap e21: 3 total, 3 up, 3 in 2024-09-16T23:54:22.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:21 smithi007 ceph-mon[23721]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:22.100 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 16 23:54:21 smithi007 sudo[43208]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T23:54:22.100 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 16 23:54:21 smithi007 sudo[43208]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:22.100 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 16 23:54:21 smithi007 sudo[43208]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:22.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-16T23:54:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:21 smithi194 ceph-mon[25713]: osdmap e21: 3 total, 3 up, 3 in 2024-09-16T23:54:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:21 smithi194 ceph-mon[25713]: pgmap v75: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:22.470 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 16 23:54:22 smithi007 sudo[43240]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T23:54:22.470 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 16 23:54:22 smithi007 sudo[43240]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:22.470 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 16 23:54:22 smithi007 sudo[43240]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:22.722 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 16 23:54:22 smithi007 sudo[43277]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T23:54:22.722 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 16 23:54:22 smithi007 sudo[43277]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:22.722 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 16 23:54:22 smithi007 sudo[43277]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:23.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:22 smithi007 ceph-mon[23721]: osdmap e22: 3 total, 3 up, 3 in 2024-09-16T23:54:23.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:22 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:22 smithi194 ceph-mon[25713]: osdmap e22: 3 total, 3 up, 3 in 2024-09-16T23:54:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:22 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:23.349 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43325]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T23:54:23.349 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43325]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:23.349 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43325]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:23.701 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43438]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T23:54:23.701 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43438]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:23.701 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43438]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:24.323 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43470]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T23:54:24.324 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43470]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:24.324 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 16 23:54:23 smithi007 sudo[43470]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:24.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 sudo[43506]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T23:54:24.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 sudo[43506]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:24.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 sudo[43506]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:24.904 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 sudo[43521]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 sudo[43521]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 sudo[43521]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 ceph-mon[23721]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 ceph-mon[23721]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:54:24.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:24 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:24 smithi194 ceph-mon[25713]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:24 smithi194 ceph-mon[25713]: from='client.14292 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi007:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:24 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 sudo[28035]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 sudo[28035]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:25.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 sudo[28035]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:25.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 sudo[28038]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-16T23:54:25.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 sudo[28038]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-16T23:54:25.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 sudo[28038]: pam_unix(sudo:session): session closed for user root 2024-09-16T23:54:26.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: mgrmap e19: smithi007.czdphd(active, since 2m), standbys: smithi194.xlteft 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T23:54:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:25 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:26.263 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T23:54:26.263 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: mgrmap e19: smithi007.czdphd(active, since 2m), standbys: smithi194.xlteft 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-16T23:54:26.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:25 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi007 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi007 to 4289M 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3939969870' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244"}]: dispatch 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3939969870' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244"}]': finished 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: osdmap e23: 4 total, 3 up, 4 in 2024-09-16T23:54:27.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:27 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:28.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi007 2024-09-16T23:54:28.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:28.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:28.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi007 to 4289M 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3939969870' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244"}]: dispatch 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3939969870' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244"}]': finished 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: osdmap e23: 4 total, 3 up, 4 in 2024-09-16T23:54:28.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:27 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:28.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:28 smithi194 ceph-mon[25713]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:28.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:28 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/11461983' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:54:29.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:28 smithi007 ceph-mon[23721]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:29.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:28 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/11461983' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:54:30.789 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:30 smithi007 ceph-mon[23721]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-16T23:54:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:30 smithi194 ceph-mon[25713]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-16T23:54:32.722 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:32 smithi007 ceph-mon[23721]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-16T23:54:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:32 smithi194 ceph-mon[25713]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-16T23:54:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:34 smithi194 ceph-mon[25713]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T23:54:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:34 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-16T23:54:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:34 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:34.925 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:34 smithi007 ceph-mon[23721]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T23:54:34.925 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:34 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-16T23:54:34.925 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:34 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:35.903 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:35 smithi007 ceph-mon[23721]: Deploying daemon osd.3 on smithi007 2024-09-16T23:54:35.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:35 smithi194 ceph-mon[25713]: Deploying daemon osd.3 on smithi007 2024-09-16T23:54:36.886 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:36 smithi007 ceph-mon[23721]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T23:54:36.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:36 smithi194 ceph-mon[25713]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T23:54:38.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:38 smithi007 ceph-mon[23721]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T23:54:38.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:38 smithi194 ceph-mon[25713]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-16T23:54:40.813 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:40 smithi007 ceph-mon[23721]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:40.813 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:40 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:40.813 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:40 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:40.813 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:40 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:40.813 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:40 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:40 smithi194 ceph-mon[25713]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:40 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:40 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:40 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:40 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:42.279 INFO:teuthology.orchestra.run.smithi007.stdout:Created osd(s) 3 on host 'smithi007' 2024-09-16T23:54:42.564 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:42 smithi007 ceph-mon[23721]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:42.564 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:42.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:42 smithi194 ceph-mon[25713]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:43.188 DEBUG:teuthology.orchestra.run.smithi007:osd.3> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.3.service 2024-09-16T23:54:43.193 INFO:tasks.cephadm:Deploying osd.4 on smithi194 with /dev/vg_nvme/lv_4... 2024-09-16T23:54:43.193 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-16T23:54:43.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:43 smithi194 ceph-mon[25713]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-16T23:54:43.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:43 smithi007 ceph-mon[23721]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-16T23:54:44.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-mon[23721]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:44.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-mon[23721]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T23:54:44.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-mon[23721]: osdmap e24: 4 total, 3 up, 4 in 2024-09-16T23:54:44.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:44.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-mon[23721]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:54:44.466 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[45093]: 2024-09-16T23:54:44.463+0000 7f07eee39700 -1 osd.3 0 waiting for initial osdmap 2024-09-16T23:54:44.467 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 16 23:54:44 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[45093]: 2024-09-16T23:54:44.468+0000 7f07eb7d4700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T23:54:44.605 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:44 smithi194 ceph-mon[25713]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:44.605 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:44 smithi194 ceph-mon[25713]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-16T23:54:44.605 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:44 smithi194 ceph-mon[25713]: osdmap e24: 4 total, 3 up, 4 in 2024-09-16T23:54:44.605 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:44 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:44.605 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:44 smithi194 ceph-mon[25713]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-16T23:54:45.641 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:45 smithi194 ceph-mon[25713]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:54:45.641 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:45 smithi194 ceph-mon[25713]: osdmap e25: 4 total, 3 up, 4 in 2024-09-16T23:54:45.641 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:45.641 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:45.646 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:54:45.672 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi194:vg_nvme/lv_4 2024-09-16T23:54:45.745 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:45 smithi007 ceph-mon[23721]: from='osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]': finished 2024-09-16T23:54:45.746 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:45 smithi007 ceph-mon[23721]: osdmap e25: 4 total, 3 up, 4 in 2024-09-16T23:54:45.746 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:45.746 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:46.613 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:54:46.613 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:54:46.613 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:46.613 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900] boot 2024-09-16T23:54:46.613 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: osdmap e26: 4 total, 4 up, 4 in 2024-09-16T23:54:46.613 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:46.614 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: osd.3 [v2:172.21.15.7:6826/472230900,v1:172.21.15.7:6827/472230900] boot 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: osdmap e26: 4 total, 4 up, 4 in 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-16T23:54:46.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:47.565 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:47 smithi194 ceph-mon[25713]: osdmap e27: 4 total, 4 up, 4 in 2024-09-16T23:54:47.815 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:47 smithi007 ceph-mon[23721]: osdmap e27: 4 total, 4 up, 4 in 2024-09-16T23:54:48.780 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:48 smithi194 ceph-mon[25713]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:48.780 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:48 smithi194 ceph-mon[25713]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:54:48.780 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:54:48.780 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:54:48.780 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:48.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:48 smithi007 ceph-mon[23721]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:48.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:48 smithi007 ceph-mon[23721]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:54:48.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:54:48.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:54:48.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi007 2024-09-16T23:54:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:50.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi007 to 3217M 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/2509484874' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4da9bf66-cdfc-4620-96ac-848fd3a021a4"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4da9bf66-cdfc-4620-96ac-848fd3a021a4"}]: dispatch 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4da9bf66-cdfc-4620-96ac-848fd3a021a4"}]': finished 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: osdmap e28: 5 total, 4 up, 5 in 2024-09-16T23:54:50.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:54:50.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:50 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/1550684535' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:54:51.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi007 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi007 to 3217M 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:54:51.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:54:51.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/2509484874' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4da9bf66-cdfc-4620-96ac-848fd3a021a4"}]: dispatch 2024-09-16T23:54:51.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4da9bf66-cdfc-4620-96ac-848fd3a021a4"}]: dispatch 2024-09-16T23:54:51.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4da9bf66-cdfc-4620-96ac-848fd3a021a4"}]': finished 2024-09-16T23:54:51.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: osdmap e28: 5 total, 4 up, 5 in 2024-09-16T23:54:51.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:54:51.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:50 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/1550684535' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:54:52.837 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:52 smithi194 ceph-mon[25713]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:53.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:52 smithi007 ceph-mon[23721]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:54.730 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:54 smithi194 ceph-mon[25713]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:55.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:54 smithi007 ceph-mon[23721]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:56.088 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:55 smithi194 ceph-mon[25713]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:56.088 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:55 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-16T23:54:56.089 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:55 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:56.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:55 smithi007 ceph-mon[23721]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:56.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:55 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-16T23:54:56.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:55 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:54:57.148 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:56 smithi194 ceph-mon[25713]: Deploying daemon osd.4 on smithi194 2024-09-16T23:54:57.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:56 smithi007 ceph-mon[23721]: Deploying daemon osd.4 on smithi194 2024-09-16T23:54:58.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:54:57 smithi194 ceph-mon[25713]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:54:58.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:54:57 smithi007 ceph-mon[23721]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:00.384 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:00 smithi194 ceph-mon[25713]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:00.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:00 smithi007 ceph-mon[23721]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:01.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:01.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:01.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:01.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:01 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:02.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:01 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:02.901 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:02 smithi194 ceph-mon[25713]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:03.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:02 smithi007 ceph-mon[23721]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:04.323 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 4 on host 'smithi194' 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='osd.4 [v2:172.21.15.194:6800/3982341081,v1:172.21.15.194:6801/3982341081]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:04.896 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:04.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:04.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='osd.4 [v2:172.21.15.194:6800/3982341081,v1:172.21.15.194:6801/3982341081]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T23:55:04.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-16T23:55:04.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:04.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:04.917 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:04.917 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:04.917 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:04.917 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:04.919 DEBUG:teuthology.orchestra.run.smithi194:osd.4> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.4.service 2024-09-16T23:55:04.921 INFO:tasks.cephadm:Deploying osd.5 on smithi194 with /dev/vg_nvme/lv_3... 2024-09-16T23:55:04.922 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-16T23:55:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:05 smithi007 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T23:55:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:05 smithi007 ceph-mon[23721]: osdmap e29: 5 total, 4 up, 5 in 2024-09-16T23:55:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:05 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:05 smithi007 ceph-mon[23721]: from='osd.4 [v2:172.21.15.194:6800/3982341081,v1:172.21.15.194:6801/3982341081]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:05 smithi007 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-mon[25713]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-16T23:55:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-mon[25713]: osdmap e29: 5 total, 4 up, 5 in 2024-09-16T23:55:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-mon[25713]: from='osd.4 [v2:172.21.15.194:6800/3982341081,v1:172.21.15.194:6801/3982341081]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-mon[25713]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:06.160 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[29939]: 2024-09-16T23:55:05.665+0000 7f6490d90700 -1 osd.4 0 waiting for initial osdmap 2024-09-16T23:55:06.160 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 16 23:55:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[29939]: 2024-09-16T23:55:05.669+0000 7f648b727700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T23:55:06.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:06 smithi194 ceph-mon[25713]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:06.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:06 smithi194 ceph-mon[25713]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:55:06.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:06 smithi194 ceph-mon[25713]: osdmap e30: 5 total, 4 up, 5 in 2024-09-16T23:55:06.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:06 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:06.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:06 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:07.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:06 smithi007 ceph-mon[23721]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:07.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:06 smithi007 ceph-mon[23721]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:55:07.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:06 smithi007 ceph-mon[23721]: osdmap e30: 5 total, 4 up, 5 in 2024-09-16T23:55:07.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:06 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:07.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:06 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:08.011 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:07 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:55:08.011 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:07 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:55:08.011 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:07 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:08.011 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:07 smithi194 ceph-mon[25713]: osd.4 [v2:172.21.15.194:6800/3982341081,v1:172.21.15.194:6801/3982341081] boot 2024-09-16T23:55:08.011 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:07 smithi194 ceph-mon[25713]: osdmap e31: 5 total, 5 up, 5 in 2024-09-16T23:55:08.011 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:07 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:08.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:07 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:55:08.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:07 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:55:08.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:07 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:08.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:07 smithi007 ceph-mon[23721]: osd.4 [v2:172.21.15.194:6800/3982341081,v1:172.21.15.194:6801/3982341081] boot 2024-09-16T23:55:08.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:07 smithi007 ceph-mon[23721]: osdmap e31: 5 total, 5 up, 5 in 2024-09-16T23:55:08.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:07 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-16T23:55:09.001 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:08 smithi194 ceph-mon[25713]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:09.002 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:08 smithi194 ceph-mon[25713]: osdmap e32: 5 total, 5 up, 5 in 2024-09-16T23:55:09.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:08 smithi007 ceph-mon[23721]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-16T23:55:09.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:08 smithi007 ceph-mon[23721]: osdmap e32: 5 total, 5 up, 5 in 2024-09-16T23:55:09.877 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: osdmap e33: 5 total, 5 up, 5 in 2024-09-16T23:55:09.877 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:09.877 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:55:09.877 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:09.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:09.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:55:09.878 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:55:10.068 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:55:10.087 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi194:vg_nvme/lv_3 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: osdmap e33: 5 total, 5 up, 5 in 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:55:10.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:55:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:10 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi194 2024-09-16T23:55:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:10 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi194 to 15936M 2024-09-16T23:55:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:10 smithi194 ceph-mon[25713]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:10 smithi194 ceph-mon[25713]: osdmap e34: 5 total, 5 up, 5 in 2024-09-16T23:55:11.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:10 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi194 2024-09-16T23:55:11.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:10 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi194 to 15936M 2024-09-16T23:55:11.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:10 smithi007 ceph-mon[23721]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:11.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:10 smithi007 ceph-mon[23721]: osdmap e34: 5 total, 5 up, 5 in 2024-09-16T23:55:11.952 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:11 smithi194 ceph-mon[25713]: osdmap e35: 5 total, 5 up, 5 in 2024-09-16T23:55:12.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:11 smithi007 ceph-mon[23721]: osdmap e35: 5 total, 5 up, 5 in 2024-09-16T23:55:12.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:12 smithi194 ceph-mon[25713]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:12.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:12 smithi194 ceph-mon[25713]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:55:12.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:12 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:55:12.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:12 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:55:12.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:12 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:12 smithi007 ceph-mon[23721]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:12 smithi007 ceph-mon[23721]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:55:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:12 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:55:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:12 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:55:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:12 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:13.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:13 smithi194 ceph-mon[25713]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-16T23:55:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:13 smithi007 ceph-mon[23721]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-16T23:55:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:14 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/4013535828' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc079723-85f2-4e3d-9b95-35949f7bed75"}]: dispatch 2024-09-16T23:55:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:14 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc079723-85f2-4e3d-9b95-35949f7bed75"}]: dispatch 2024-09-16T23:55:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:14 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc079723-85f2-4e3d-9b95-35949f7bed75"}]': finished 2024-09-16T23:55:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:14 smithi007 ceph-mon[23721]: osdmap e36: 6 total, 5 up, 6 in 2024-09-16T23:55:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:14 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:14 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/4013535828' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc079723-85f2-4e3d-9b95-35949f7bed75"}]: dispatch 2024-09-16T23:55:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:14 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bc079723-85f2-4e3d-9b95-35949f7bed75"}]: dispatch 2024-09-16T23:55:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:14 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bc079723-85f2-4e3d-9b95-35949f7bed75"}]': finished 2024-09-16T23:55:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:14 smithi194 ceph-mon[25713]: osdmap e36: 6 total, 5 up, 6 in 2024-09-16T23:55:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:14 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:15 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/2648798601' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:55:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:15 smithi007 ceph-mon[23721]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T23:55:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:15 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/2648798601' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:55:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:15 smithi194 ceph-mon[25713]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T23:55:18.373 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:18 smithi194 ceph-mon[25713]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-16T23:55:18.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:18 smithi007 ceph-mon[23721]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-16T23:55:20.381 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:20 smithi194 ceph-mon[25713]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-16T23:55:20.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:20 smithi007 ceph-mon[23721]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-16T23:55:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-16T23:55:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:21 smithi194 ceph-mon[25713]: Deploying daemon osd.5 on smithi194 2024-09-16T23:55:21.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-16T23:55:21.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:21.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:21 smithi007 ceph-mon[23721]: Deploying daemon osd.5 on smithi194 2024-09-16T23:55:22.152 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:22 smithi194 ceph-mon[25713]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T23:55:22.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:22 smithi007 ceph-mon[23721]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T23:55:24.268 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:24 smithi194 ceph-mon[25713]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:24.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:24 smithi007 ceph-mon[23721]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:26.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:26 smithi007 ceph-mon[23721]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:26.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:26 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:26.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:26 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:26.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:26 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:26.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:26 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:26.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:26 smithi194 ceph-mon[25713]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:26.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:26 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:26.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:26 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:26.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:26 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:26.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:26 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:28.570 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:28 smithi194 ceph-mon[25713]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:28.601 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 5 on host 'smithi194' 2024-09-16T23:55:28.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:28 smithi007 ceph-mon[23721]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:29.851 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:29 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:29.851 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:29 smithi194 ceph-mon[25713]: from='osd.5 [v2:172.21.15.194:6808/636284694,v1:172.21.15.194:6809/636284694]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T23:55:29.852 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:29 smithi194 ceph-mon[25713]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T23:55:29.874 DEBUG:teuthology.orchestra.run.smithi194:osd.5> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.5.service 2024-09-16T23:55:29.876 INFO:tasks.cephadm:Deploying osd.6 on smithi194 with /dev/vg_nvme/lv_2... 2024-09-16T23:55:29.876 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-16T23:55:30.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:29 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:30.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:29 smithi007 ceph-mon[23721]: from='osd.5 [v2:172.21.15.194:6808/636284694,v1:172.21.15.194:6809/636284694]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T23:55:30.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:29 smithi007 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: osdmap e37: 6 total, 5 up, 6 in 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: from='osd.5 [v2:172.21.15.194:6808/636284694,v1:172.21.15.194:6809/636284694]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:30.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:30.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[32896]: 2024-09-16T23:55:30.617+0000 7f18f5299700 -1 osd.5 0 waiting for initial osdmap 2024-09-16T23:55:30.910 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 16 23:55:30 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[32896]: 2024-09-16T23:55:30.622+0000 7f18efc30700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: osdmap e37: 6 total, 5 up, 6 in 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: from='osd.5 [v2:172.21.15.194:6808/636284694,v1:172.21.15.194:6809/636284694]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:31.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:30 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:31 smithi194 ceph-mon[25713]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:55:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:31 smithi194 ceph-mon[25713]: osdmap e38: 6 total, 5 up, 6 in 2024-09-16T23:55:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:31 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:31 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:31 smithi007 ceph-mon[23721]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:55:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:31 smithi007 ceph-mon[23721]: osdmap e38: 6 total, 5 up, 6 in 2024-09-16T23:55:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:31 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:31 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:32.673 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:55:32.693 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi194:vg_nvme/lv_2 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: osd.5 [v2:172.21.15.194:6808/636284694,v1:172.21.15.194:6809/636284694] boot 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: osdmap e39: 6 total, 6 up, 6 in 2024-09-16T23:55:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:32 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: osd.5 [v2:172.21.15.194:6808/636284694,v1:172.21.15.194:6809/636284694] boot 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: osdmap e39: 6 total, 6 up, 6 in 2024-09-16T23:55:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:32 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-16T23:55:33.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:33 smithi194 ceph-mon[25713]: osdmap e40: 6 total, 6 up, 6 in 2024-09-16T23:55:34.000 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:33 smithi007 ceph-mon[23721]: osdmap e40: 6 total, 6 up, 6 in 2024-09-16T23:55:34.839 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:34 smithi194 ceph-mon[25713]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:34.839 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:34 smithi194 ceph-mon[25713]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T23:55:34.839 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:34 smithi194 ceph-mon[25713]: osdmap e41: 6 total, 6 up, 6 in 2024-09-16T23:55:35.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:34 smithi007 ceph-mon[23721]: pgmap v130: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:35.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:34 smithi007 ceph-mon[23721]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-16T23:55:35.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:34 smithi007 ceph-mon[23721]: osdmap e41: 6 total, 6 up, 6 in 2024-09-16T23:55:35.736 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:35 smithi194 ceph-mon[25713]: osdmap e42: 6 total, 6 up, 6 in 2024-09-16T23:55:35.736 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:35 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:55:35.737 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:35 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:55:35.737 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:35 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:36.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:35 smithi007 ceph-mon[23721]: osdmap e42: 6 total, 6 up, 6 in 2024-09-16T23:55:36.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:35 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:55:36.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:35 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:55:36.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:35 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:36.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:36 smithi194 ceph-mon[25713]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:55:36.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:36 smithi194 ceph-mon[25713]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:36.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:36 smithi194 ceph-mon[25713]: osdmap e43: 6 total, 6 up, 6 in 2024-09-16T23:55:37.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:36 smithi007 ceph-mon[23721]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:55:37.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:36 smithi007 ceph-mon[23721]: pgmap v133: 1 pgs: 1 peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:37.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:36 smithi007 ceph-mon[23721]: osdmap e43: 6 total, 6 up, 6 in 2024-09-16T23:55:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: pgmap v135: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi194 2024-09-16T23:55:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:55:38.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi194 to 7968M 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/3613682543' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a21b541-5d12-4cff-b2c1-990a294ce2d7"}]: dispatch 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a21b541-5d12-4cff-b2c1-990a294ce2d7"}]: dispatch 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a21b541-5d12-4cff-b2c1-990a294ce2d7"}]': finished 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: osdmap e44: 7 total, 6 up, 7 in 2024-09-16T23:55:38.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:38 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: pgmap v135: 1 pgs: 1 peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi194 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi194 to 7968M 2024-09-16T23:55:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/3613682543' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a21b541-5d12-4cff-b2c1-990a294ce2d7"}]: dispatch 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7a21b541-5d12-4cff-b2c1-990a294ce2d7"}]: dispatch 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7a21b541-5d12-4cff-b2c1-990a294ce2d7"}]': finished 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: osdmap e44: 7 total, 6 up, 7 in 2024-09-16T23:55:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:38 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:39 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/3379421232' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:55:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:39 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:39.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:39 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/3379421232' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:55:39.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:39 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:40.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:40 smithi194 ceph-mon[25713]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-16T23:55:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:40 smithi194 ceph-mon[25713]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T23:55:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:40 smithi194 ceph-mon[25713]: Cluster is now healthy 2024-09-16T23:55:40.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:40 smithi007 ceph-mon[23721]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-09-16T23:55:40.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:40 smithi007 ceph-mon[23721]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-16T23:55:40.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:40 smithi007 ceph-mon[23721]: Cluster is now healthy 2024-09-16T23:55:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:42 smithi194 ceph-mon[25713]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-16T23:55:42.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:42 smithi007 ceph-mon[23721]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-16T23:55:44.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:44 smithi194 ceph-mon[25713]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T23:55:44.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:44 smithi007 ceph-mon[23721]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T23:55:45.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-16T23:55:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:45 smithi194 ceph-mon[25713]: Deploying daemon osd.6 on smithi194 2024-09-16T23:55:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-16T23:55:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:45 smithi007 ceph-mon[23721]: Deploying daemon osd.6 on smithi194 2024-09-16T23:55:46.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:46 smithi194 ceph-mon[25713]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-16T23:55:46.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:46 smithi007 ceph-mon[23721]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-09-16T23:55:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:48 smithi194 ceph-mon[25713]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T23:55:48.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:48 smithi007 ceph-mon[23721]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T23:55:50.585 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:50 smithi194 ceph-mon[25713]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:50.585 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:50.585 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:50.585 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:50.585 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:50.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:50 smithi007 ceph-mon[23721]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:50.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:50.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:55:50.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:55:50.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:55:52.336 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:52 smithi194 ceph-mon[25713]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:52.368 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 6 on host 'smithi194' 2024-09-16T23:55:52.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:52 smithi007 ceph-mon[23721]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:52.949 DEBUG:teuthology.orchestra.run.smithi194:osd.6> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.6.service 2024-09-16T23:55:52.951 INFO:tasks.cephadm:Deploying osd.7 on smithi194 with /dev/vg_nvme/lv_1... 2024-09-16T23:55:52.951 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-16T23:55:53.578 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:53 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:53.578 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:53 smithi194 ceph-mon[25713]: from='osd.6 [v2:172.21.15.194:6816/435360745,v1:172.21.15.194:6817/435360745]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T23:55:53.578 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:53 smithi194 ceph-mon[25713]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T23:55:53.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:53 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:53.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:53 smithi007 ceph-mon[23721]: from='osd.6 [v2:172.21.15.194:6816/435360745,v1:172.21.15.194:6817/435360745]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T23:55:53.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:53 smithi007 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-16T23:55:54.624 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:54.624 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T23:55:54.624 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: osdmap e45: 7 total, 6 up, 7 in 2024-09-16T23:55:54.624 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:54.624 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: from='osd.6 [v2:172.21.15.194:6816/435360745,v1:172.21.15.194:6817/435360745]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:54.624 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:54.625 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:54.625 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[35912]: 2024-09-16T23:55:54.388+0000 7fdf3ff9a700 -1 osd.6 0 waiting for initial osdmap 2024-09-16T23:55:54.625 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 16 23:55:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[35912]: 2024-09-16T23:55:54.394+0000 7fdf3b132700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: osdmap e45: 7 total, 6 up, 7 in 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: from='osd.6 [v2:172.21.15.194:6816/435360745,v1:172.21.15.194:6817/435360745]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:55:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:55.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:55:55.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: osdmap e46: 7 total, 6 up, 7 in 2024-09-16T23:55:55.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:55.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:55.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:55.501 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: osd.6 [v2:172.21.15.194:6816/435360745,v1:172.21.15.194:6817/435360745] boot 2024-09-16T23:55:55.502 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: osdmap e47: 7 total, 7 up, 7 in 2024-09-16T23:55:55.502 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:55 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:55:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: osdmap e46: 7 total, 6 up, 7 in 2024-09-16T23:55:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:55:55.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: osd.6 [v2:172.21.15.194:6816/435360745,v1:172.21.15.194:6817/435360745] boot 2024-09-16T23:55:55.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: osdmap e47: 7 total, 7 up, 7 in 2024-09-16T23:55:55.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:55 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-16T23:55:56.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:56 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:55:56.596 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:56 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:55:56.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:56 smithi194 ceph-mon[25713]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:56.597 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:56 smithi194 ceph-mon[25713]: osdmap e48: 7 total, 7 up, 7 in 2024-09-16T23:55:56.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:56 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:55:56.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:56 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:55:56.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:56 smithi007 ceph-mon[23721]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-16T23:55:56.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:56 smithi007 ceph-mon[23721]: osdmap e48: 7 total, 7 up, 7 in 2024-09-16T23:55:56.867 INFO:teuthology.orchestra.run.smithi194.stdout: 2024-09-16T23:55:56.893 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi194:vg_nvme/lv_1 2024-09-16T23:55:58.131 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:57 smithi194 ceph-mon[25713]: osdmap e49: 7 total, 7 up, 7 in 2024-09-16T23:55:58.131 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:57 smithi194 ceph-mon[25713]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:55:58.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:57 smithi007 ceph-mon[23721]: osdmap e49: 7 total, 7 up, 7 in 2024-09-16T23:55:58.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:57 smithi007 ceph-mon[23721]: pgmap v151: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:55:58.976 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:58 smithi194 ceph-mon[25713]: osdmap e50: 7 total, 7 up, 7 in 2024-09-16T23:55:59.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:58 smithi007 ceph-mon[23721]: osdmap e50: 7 total, 7 up, 7 in 2024-09-16T23:56:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: osdmap e51: 7 total, 7 up, 7 in 2024-09-16T23:56:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:56:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:56:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:56:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi194 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi194 to 5312M 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:56:00.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:55:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: osdmap e51: 7 total, 7 up, 7 in 2024-09-16T23:56:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='client.24211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi194:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:56:00.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi194 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi194 to 5312M 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:00.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:56:00.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:55:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-16T23:56:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/1785749300' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "428284d0-2ee1-4431-ab27-0e5405f6e316"}]: dispatch 2024-09-16T23:56:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "428284d0-2ee1-4431-ab27-0e5405f6e316"}]: dispatch 2024-09-16T23:56:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "428284d0-2ee1-4431-ab27-0e5405f6e316"}]': finished 2024-09-16T23:56:02.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: osdmap e52: 8 total, 7 up, 8 in 2024-09-16T23:56:02.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:02.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:02 smithi194 ceph-mon[25713]: from='client.? 172.21.15.194:0/1851751025' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/1785749300' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "428284d0-2ee1-4431-ab27-0e5405f6e316"}]: dispatch 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "428284d0-2ee1-4431-ab27-0e5405f6e316"}]: dispatch 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "428284d0-2ee1-4431-ab27-0e5405f6e316"}]': finished 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: osdmap e52: 8 total, 7 up, 8 in 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:02.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:02 smithi007 ceph-mon[23721]: from='client.? 172.21.15.194:0/1851751025' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-16T23:56:04.401 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:04 smithi007 ceph-mon[23721]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T23:56:04.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:04 smithi194 ceph-mon[25713]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T23:56:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:05 smithi194 ceph-mon[25713]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-16T23:56:06.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:05 smithi007 ceph-mon[23721]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-16T23:56:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:08 smithi194 ceph-mon[25713]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-16T23:56:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-16T23:56:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:08 smithi007 ceph-mon[23721]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-09-16T23:56:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-16T23:56:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:09.328 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:09 smithi194 ceph-mon[25713]: Deploying daemon osd.7 on smithi194 2024-09-16T23:56:09.328 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:56:09.328 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:56:09.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:09 smithi007 ceph-mon[23721]: Deploying daemon osd.7 on smithi194 2024-09-16T23:56:09.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:56:09.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:56:10.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:10 smithi194 ceph-mon[25713]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T23:56:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:10 smithi007 ceph-mon[23721]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-09-16T23:56:12.378 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:12 smithi194 ceph-mon[25713]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:12.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:12 smithi007 ceph-mon[23721]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:56:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:14.094 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:56:14.095 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:13 smithi194 ceph-mon[25713]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:56:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:56:14.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:13 smithi007 ceph-mon[23721]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:15 smithi007 ceph-mon[23721]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:15 smithi007 ceph-mon[23721]: from='osd.7 [v2:172.21.15.194:6824/868693308,v1:172.21.15.194:6825/868693308]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T23:56:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:15 smithi007 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T23:56:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:15 smithi194 ceph-mon[25713]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:15 smithi194 ceph-mon[25713]: from='osd.7 [v2:172.21.15.194:6824/868693308,v1:172.21.15.194:6825/868693308]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T23:56:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:15 smithi194 ceph-mon[25713]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-16T23:56:16.245 INFO:teuthology.orchestra.run.smithi194.stdout:Created osd(s) 7 on host 'smithi194' 2024-09-16T23:56:17.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:16 smithi194 ceph-mon[25713]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T23:56:17.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:16 smithi194 ceph-mon[25713]: osdmap e53: 8 total, 7 up, 8 in 2024-09-16T23:56:17.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:16 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:17.119 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:16 smithi194 ceph-mon[25713]: from='osd.7 [v2:172.21.15.194:6824/868693308,v1:172.21.15.194:6825/868693308]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:56:17.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:16 smithi194 ceph-mon[25713]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:56:17.120 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:16 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:17.141 DEBUG:teuthology.orchestra.run.smithi194:osd.7> sudo journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.7.service 2024-09-16T23:56:17.144 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-16T23:56:17.144 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-16T23:56:17.174 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:16 smithi007 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-16T23:56:17.174 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:16 smithi007 ceph-mon[23721]: osdmap e53: 8 total, 7 up, 8 in 2024-09-16T23:56:17.174 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:16 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:17.174 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:16 smithi007 ceph-mon[23721]: from='osd.7 [v2:172.21.15.194:6824/868693308,v1:172.21.15.194:6825/868693308]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:56:17.174 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:16 smithi007 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-16T23:56:17.174 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:16 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:17 smithi007 ceph-mon[23721]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:56:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:17 smithi007 ceph-mon[23721]: osdmap e54: 8 total, 7 up, 8 in 2024-09-16T23:56:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:17 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:17 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:18.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:17 smithi007 ceph-mon[23721]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:18.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:17 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:18.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:17 smithi194 ceph-mon[25713]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]': finished 2024-09-16T23:56:18.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:17 smithi194 ceph-mon[25713]: osdmap e54: 8 total, 7 up, 8 in 2024-09-16T23:56:18.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:17 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:18.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:17 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:18.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:17 smithi194 ceph-mon[25713]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-16T23:56:18.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:17 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:19.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:18 smithi007 ceph-mon[23721]: purged_snaps scrub starts 2024-09-16T23:56:19.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:18 smithi007 ceph-mon[23721]: purged_snaps scrub ok 2024-09-16T23:56:19.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:18 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:19.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:18 smithi007 ceph-mon[23721]: osd.7 [v2:172.21.15.194:6824/868693308,v1:172.21.15.194:6825/868693308] boot 2024-09-16T23:56:19.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:18 smithi007 ceph-mon[23721]: osdmap e55: 8 total, 8 up, 8 in 2024-09-16T23:56:19.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:18 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:19.172 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:56:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:18 smithi194 ceph-mon[25713]: purged_snaps scrub starts 2024-09-16T23:56:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:18 smithi194 ceph-mon[25713]: purged_snaps scrub ok 2024-09-16T23:56:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:18 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:18 smithi194 ceph-mon[25713]: osd.7 [v2:172.21.15.194:6824/868693308,v1:172.21.15.194:6825/868693308] boot 2024-09-16T23:56:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:18 smithi194 ceph-mon[25713]: osdmap e55: 8 total, 8 up, 8 in 2024-09-16T23:56:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:18 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-16T23:56:19.821 INFO:teuthology.orchestra.run.smithi007.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1726530977,"num_in_osds":8,"osd_in_since":1726530961,"num_remapped_pgs":1} 2024-09-16T23:56:19.822 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T23:56:20.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:19 smithi194 ceph-mon[25713]: osdmap e56: 8 total, 8 up, 8 in 2024-09-16T23:56:20.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:19 smithi194 ceph-mon[25713]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:20.048 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:19 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2343071338' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-16T23:56:20.084 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:20.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:19 smithi007 ceph-mon[23721]: osdmap e56: 8 total, 8 up, 8 in 2024-09-16T23:56:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:19 smithi007 ceph-mon[23721]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:19 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2343071338' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-16T23:56:21.322 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: osdmap e57: 8 total, 8 up, 8 in 2024-09-16T23:56:21.322 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: Detected new or changed devices on smithi194 2024-09-16T23:56:21.322 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi194 to 3984M 2024-09-16T23:56:21.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:56:21.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:21.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:56:21.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:20 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: osdmap e57: 8 total, 8 up, 8 in 2024-09-16T23:56:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: Detected new or changed devices on smithi194 2024-09-16T23:56:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-16T23:56:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi194 to 3984M 2024-09-16T23:56:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:56:21.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:56:21.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:56:21.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:20 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:56:21.992 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:56:21.992 INFO:teuthology.orchestra.run.smithi007.stdout:{"epoch":58,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","created":"2024-09-16T23:50:24.615556+0000","modified":"2024-09-16T23:56:21.399944+0000","last_up_change":"2024-09-16T23:56:17.908134+0000","last_in_change":"2024-09-16T23:56:01.478447+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T23:54:19.546870+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b298b2c2-3864-42e7-bf52-1ffbf6adb517","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6802","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6803","nonce":3763790667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6804","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6805","nonce":3763790667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6808","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6809","nonce":3763790667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6806","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6807","nonce":3763790667}]},"public_addr":"172.21.15.7:6803/3763790667","cluster_addr":"172.21.15.7:6805/3763790667","heartbeat_back_addr":"172.21.15.7:6809/3763790667","heartbeat_front_addr":"172.21.15.7:6807/3763790667","state":["exists","up"]},{"osd":1,"uuid":"e2a2d65a-c40b-45ba-b2a9-051d7301c685","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6810","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6811","nonce":3862868244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6812","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6813","nonce":3862868244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6816","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6817","nonce":3862868244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6814","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6815","nonce":3862868244}]},"public_addr":"172.21.15.7:6811/3862868244","cluster_addr":"172.21.15.7:6813/3862868244","heartbeat_back_addr":"172.21.15.7:6817/3862868244","heartbeat_front_addr":"172.21.15.7:6815/3862868244","state":["exists","up"]},{"osd":2,"uuid":"7225a34e-aff2-4991-a631-d895e13a92ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6818","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6819","nonce":3183589106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6820","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6821","nonce":3183589106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6824","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6825","nonce":3183589106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6822","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6823","nonce":3183589106}]},"public_addr":"172.21.15.7:6819/3183589106","cluster_addr":"172.21.15.7:6821/3183589106","heartbeat_back_addr":"172.21.15.7:6825/3183589106","heartbeat_front_addr":"172.21.15.7:6823/3183589106","state":["exists","up"]},{"osd":3,"uuid":"7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6826","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6827","nonce":472230900}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6828","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6829","nonce":472230900}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6832","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6833","nonce":472230900}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6830","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6831","nonce":472230900}]},"public_addr":"172.21.15.7:6827/472230900","cluster_addr":"172.21.15.7:6829/472230900","heartbeat_back_addr":"172.21.15.7:6833/472230900","heartbeat_front_addr":"172.21.15.7:6831/472230900","state":["exists","up"]},{"osd":4,"uuid":"4da9bf66-cdfc-4620-96ac-848fd3a021a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6801","nonce":3982341081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6803","nonce":3982341081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6807","nonce":3982341081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6805","nonce":3982341081}]},"public_addr":"172.21.15.194:6801/3982341081","cluster_addr":"172.21.15.194:6803/3982341081","heartbeat_back_addr":"172.21.15.194:6807/3982341081","heartbeat_front_addr":"172.21.15.194:6805/3982341081","state":["exists","up"]},{"osd":5,"uuid":"bc079723-85f2-4e3d-9b95-35949f7bed75","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6809","nonce":636284694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6811","nonce":636284694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6815","nonce":636284694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6813","nonce":636284694}]},"public_addr":"172.21.15.194:6809/636284694","cluster_addr":"172.21.15.194:6811/636284694","heartbeat_back_addr":"172.21.15.194:6815/636284694","heartbeat_front_addr":"172.21.15.194:6813/636284694","state":["exists","up"]},{"osd":6,"uuid":"7a21b541-5d12-4cff-b2c1-990a294ce2d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6817","nonce":435360745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6819","nonce":435360745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6823","nonce":435360745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6821","nonce":435360745}]},"public_addr":"172.21.15.194:6817/435360745","cluster_addr":"172.21.15.194:6819/435360745","heartbeat_back_addr":"172.21.15.194:6823/435360745","heartbeat_front_addr":"172.21.15.194:6821/435360745","state":["exists","up"]},{"osd":7,"uuid":"428284d0-2ee1-4431-ab27-0e5405f6e316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6825","nonce":868693308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6827","nonce":868693308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6831","nonce":868693308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6829","nonce":868693308}]},"public_addr":"172.21.15.194:6825/868693308","cluster_addr":"172.21.15.194:6827/868693308","heartbeat_back_addr":"172.21.15.194:6831/868693308","heartbeat_front_addr":"172.21.15.194:6829/868693308","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:53:18.086003+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:53:49.882579+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:54:16.844354+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:54:43.595494+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:04.781571+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:29.654845+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:53.380423+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:56:16.831974+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.7:6801/2587638492":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/310194880":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/1001861505":"2024-09-17T23:52:08.975108+0000","172.21.15.7:6800/2587638492":"2024-09-17T23:52:08.975108+0000","172.21.15.7:6801/2537923547":"2024-09-17T23:51:20.004938+0000","172.21.15.7:6800/2537923547":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/1404750921":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/955256528":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/3720859820":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/3879771779":"2024-09-17T23:50:51.103747+0000","172.21.15.7:6800/3263209100":"2024-09-17T23:50:51.103747+0000","172.21.15.7:6801/3263209100":"2024-09-17T23:50:51.103747+0000","172.21.15.7:0/2931702999":"2024-09-17T23:50:51.103747+0000","172.21.15.7:0/604101463":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/1304049360":"2024-09-17T23:50:51.103747+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T23:56:22.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:22 smithi007 ceph-mon[23721]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:22.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:22 smithi007 ceph-mon[23721]: osdmap e58: 8 total, 8 up, 8 in 2024-09-16T23:56:22.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:22 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/954205912' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T23:56:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:22 smithi194 ceph-mon[25713]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:22 smithi194 ceph-mon[25713]: osdmap e58: 8 total, 8 up, 8 in 2024-09-16T23:56:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:22 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/954205912' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T23:56:22.663 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-16T23:54:19.546870+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-16T23:56:22.663 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-16T23:56:22.886 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:23.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:23 smithi007 ceph-mon[23721]: osdmap e59: 8 total, 8 up, 8 in 2024-09-16T23:56:23.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:23 smithi194 ceph-mon[25713]: osdmap e59: 8 total, 8 up, 8 in 2024-09-16T23:56:24.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:24 smithi007 ceph-mon[23721]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-09-16T23:56:24.707 INFO:teuthology.orchestra.run.smithi007.stdout:pg_num: 1 2024-09-16T23:56:24.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:24 smithi194 ceph-mon[25713]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 124 KiB/s, 0 objects/s recovering 2024-09-16T23:56:25.407 INFO:tasks.cephadm:Setting up client nodes... 2024-09-16T23:56:25.408 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-16T23:56:25.713 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:25 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1316522253' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-16T23:56:25.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:25 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1316522253' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-16T23:56:26.843 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:26 smithi007 ceph-mon[23721]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T23:56:26.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:26 smithi194 ceph-mon[25713]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-16T23:56:27.608 INFO:teuthology.orchestra.run.smithi007.stdout:[client.0] 2024-09-16T23:56:27.608 INFO:teuthology.orchestra.run.smithi007.stdout: key = AQCrxehm8tYzJBAAAup0m2ZkJ9O5LIfHUn/O8w== 2024-09-16T23:56:28.372 DEBUG:teuthology.orchestra.run.smithi007:> set -ex 2024-09-16T23:56:28.372 DEBUG:teuthology.orchestra.run.smithi007:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-16T23:56:28.372 DEBUG:teuthology.orchestra.run.smithi007:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-16T23:56:28.415 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-16T23:56:28.415 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-16T23:56:28.416 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-16T23:56:28.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:28 smithi007 ceph-mon[23721]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-16T23:56:28.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:28 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1055129056' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-16T23:56:28.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:28 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1055129056' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-16T23:56:28.672 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:28.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:28 smithi194 ceph-mon[25713]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-16T23:56:28.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:28 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1055129056' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-16T23:56:28.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:28 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1055129056' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-16T23:56:30.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:30 smithi007 ceph-mon[23721]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T23:56:30.689 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:56:30.711 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:30 smithi194 ceph-mon[25713]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-16T23:56:31.654 INFO:teuthology.orchestra.run.smithi007.stdout:{"epoch":19,"active_gid":14200,"active_name":"smithi007.czdphd","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6800","nonce":2763609584},{"type":"v1","addr":"172.21.15.7:6801","nonce":2763609584}]},"active_addr":"172.21.15.7:6801/2763609584","active_change":"2024-09-16T23:52:08.975491+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14220,"name":"smithi194.xlteft","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.7:8443/","prometheus":"http://172.21.15.7:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.7:0","nonce":2154047997}]},{"addrvec":[{"type":"v2","addr":"172.21.15.7:0","nonce":2809849119}]},{"addrvec":[{"type":"v2","addr":"172.21.15.7:0","nonce":2786563047}]},{"addrvec":[{"type":"v2","addr":"172.21.15.7:0","nonce":297762331}]}]}} 2024-09-16T23:56:31.658 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-16T23:56:31.658 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-16T23:56:31.658 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T23:56:31.685 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:31 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/170467282' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-16T23:56:31.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:31 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/170467282' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-16T23:56:31.887 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:32.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:32 smithi007 ceph-mon[23721]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:32.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:32 smithi194 ceph-mon[25713]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:33.578 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:56:33.578 INFO:teuthology.orchestra.run.smithi007.stdout:{"epoch":59,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","created":"2024-09-16T23:50:24.615556+0000","modified":"2024-09-16T23:56:22.401462+0000","last_up_change":"2024-09-16T23:56:17.908134+0000","last_in_change":"2024-09-16T23:56:01.478447+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T23:54:19.546870+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b298b2c2-3864-42e7-bf52-1ffbf6adb517","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6802","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6803","nonce":3763790667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6804","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6805","nonce":3763790667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6808","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6809","nonce":3763790667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6806","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6807","nonce":3763790667}]},"public_addr":"172.21.15.7:6803/3763790667","cluster_addr":"172.21.15.7:6805/3763790667","heartbeat_back_addr":"172.21.15.7:6809/3763790667","heartbeat_front_addr":"172.21.15.7:6807/3763790667","state":["exists","up"]},{"osd":1,"uuid":"e2a2d65a-c40b-45ba-b2a9-051d7301c685","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6810","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6811","nonce":3862868244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6812","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6813","nonce":3862868244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6816","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6817","nonce":3862868244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6814","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6815","nonce":3862868244}]},"public_addr":"172.21.15.7:6811/3862868244","cluster_addr":"172.21.15.7:6813/3862868244","heartbeat_back_addr":"172.21.15.7:6817/3862868244","heartbeat_front_addr":"172.21.15.7:6815/3862868244","state":["exists","up"]},{"osd":2,"uuid":"7225a34e-aff2-4991-a631-d895e13a92ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6818","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6819","nonce":3183589106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6820","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6821","nonce":3183589106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6824","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6825","nonce":3183589106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6822","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6823","nonce":3183589106}]},"public_addr":"172.21.15.7:6819/3183589106","cluster_addr":"172.21.15.7:6821/3183589106","heartbeat_back_addr":"172.21.15.7:6825/3183589106","heartbeat_front_addr":"172.21.15.7:6823/3183589106","state":["exists","up"]},{"osd":3,"uuid":"7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6826","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6827","nonce":472230900}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6828","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6829","nonce":472230900}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6832","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6833","nonce":472230900}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6830","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6831","nonce":472230900}]},"public_addr":"172.21.15.7:6827/472230900","cluster_addr":"172.21.15.7:6829/472230900","heartbeat_back_addr":"172.21.15.7:6833/472230900","heartbeat_front_addr":"172.21.15.7:6831/472230900","state":["exists","up"]},{"osd":4,"uuid":"4da9bf66-cdfc-4620-96ac-848fd3a021a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6801","nonce":3982341081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6803","nonce":3982341081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6807","nonce":3982341081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6805","nonce":3982341081}]},"public_addr":"172.21.15.194:6801/3982341081","cluster_addr":"172.21.15.194:6803/3982341081","heartbeat_back_addr":"172.21.15.194:6807/3982341081","heartbeat_front_addr":"172.21.15.194:6805/3982341081","state":["exists","up"]},{"osd":5,"uuid":"bc079723-85f2-4e3d-9b95-35949f7bed75","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6809","nonce":636284694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6811","nonce":636284694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6815","nonce":636284694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6813","nonce":636284694}]},"public_addr":"172.21.15.194:6809/636284694","cluster_addr":"172.21.15.194:6811/636284694","heartbeat_back_addr":"172.21.15.194:6815/636284694","heartbeat_front_addr":"172.21.15.194:6813/636284694","state":["exists","up"]},{"osd":6,"uuid":"7a21b541-5d12-4cff-b2c1-990a294ce2d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6817","nonce":435360745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6819","nonce":435360745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6823","nonce":435360745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6821","nonce":435360745}]},"public_addr":"172.21.15.194:6817/435360745","cluster_addr":"172.21.15.194:6819/435360745","heartbeat_back_addr":"172.21.15.194:6823/435360745","heartbeat_front_addr":"172.21.15.194:6821/435360745","state":["exists","up"]},{"osd":7,"uuid":"428284d0-2ee1-4431-ab27-0e5405f6e316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6825","nonce":868693308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6827","nonce":868693308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6831","nonce":868693308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6829","nonce":868693308}]},"public_addr":"172.21.15.194:6825/868693308","cluster_addr":"172.21.15.194:6827/868693308","heartbeat_back_addr":"172.21.15.194:6831/868693308","heartbeat_front_addr":"172.21.15.194:6829/868693308","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:53:18.086003+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:53:49.882579+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:54:16.844354+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:54:43.595494+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:04.781571+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:29.654845+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:53.380423+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:56:16.831974+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.7:6801/2587638492":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/310194880":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/1001861505":"2024-09-17T23:52:08.975108+0000","172.21.15.7:6800/2587638492":"2024-09-17T23:52:08.975108+0000","172.21.15.7:6801/2537923547":"2024-09-17T23:51:20.004938+0000","172.21.15.7:6800/2537923547":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/1404750921":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/955256528":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/3720859820":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/3879771779":"2024-09-17T23:50:51.103747+0000","172.21.15.7:6800/3263209100":"2024-09-17T23:50:51.103747+0000","172.21.15.7:6801/3263209100":"2024-09-17T23:50:51.103747+0000","172.21.15.7:0/2931702999":"2024-09-17T23:50:51.103747+0000","172.21.15.7:0/604101463":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/1304049360":"2024-09-17T23:50:51.103747+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T23:56:34.369 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-16T23:56:34.369 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-16T23:56:34.427 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:34 smithi007 ceph-mon[23721]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:34.428 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:34 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3219224013' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T23:56:34.632 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:34 smithi194 ceph-mon[25713]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:34 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3219224013' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T23:56:36.012 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:35 smithi007 ceph-mon[23721]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:36.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:35 smithi194 ceph-mon[25713]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:36.575 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:56:36.575 INFO:teuthology.orchestra.run.smithi007.stdout:{"epoch":59,"fsid":"5c15e83c-7486-11ef-bceb-c7b262605968","created":"2024-09-16T23:50:24.615556+0000","modified":"2024-09-16T23:56:22.401462+0000","last_up_change":"2024-09-16T23:56:17.908134+0000","last_in_change":"2024-09-16T23:56:01.478447+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-16T23:54:19.546870+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"b298b2c2-3864-42e7-bf52-1ffbf6adb517","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6802","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6803","nonce":3763790667}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6804","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6805","nonce":3763790667}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6808","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6809","nonce":3763790667}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6806","nonce":3763790667},{"type":"v1","addr":"172.21.15.7:6807","nonce":3763790667}]},"public_addr":"172.21.15.7:6803/3763790667","cluster_addr":"172.21.15.7:6805/3763790667","heartbeat_back_addr":"172.21.15.7:6809/3763790667","heartbeat_front_addr":"172.21.15.7:6807/3763790667","state":["exists","up"]},{"osd":1,"uuid":"e2a2d65a-c40b-45ba-b2a9-051d7301c685","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6810","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6811","nonce":3862868244}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6812","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6813","nonce":3862868244}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6816","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6817","nonce":3862868244}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6814","nonce":3862868244},{"type":"v1","addr":"172.21.15.7:6815","nonce":3862868244}]},"public_addr":"172.21.15.7:6811/3862868244","cluster_addr":"172.21.15.7:6813/3862868244","heartbeat_back_addr":"172.21.15.7:6817/3862868244","heartbeat_front_addr":"172.21.15.7:6815/3862868244","state":["exists","up"]},{"osd":2,"uuid":"7225a34e-aff2-4991-a631-d895e13a92ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6818","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6819","nonce":3183589106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6820","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6821","nonce":3183589106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6824","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6825","nonce":3183589106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6822","nonce":3183589106},{"type":"v1","addr":"172.21.15.7:6823","nonce":3183589106}]},"public_addr":"172.21.15.7:6819/3183589106","cluster_addr":"172.21.15.7:6821/3183589106","heartbeat_back_addr":"172.21.15.7:6825/3183589106","heartbeat_front_addr":"172.21.15.7:6823/3183589106","state":["exists","up"]},{"osd":3,"uuid":"7f23f8c2-1b6d-4fe3-8dbf-b6b4f3408244","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6826","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6827","nonce":472230900}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6828","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6829","nonce":472230900}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6832","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6833","nonce":472230900}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.7:6830","nonce":472230900},{"type":"v1","addr":"172.21.15.7:6831","nonce":472230900}]},"public_addr":"172.21.15.7:6827/472230900","cluster_addr":"172.21.15.7:6829/472230900","heartbeat_back_addr":"172.21.15.7:6833/472230900","heartbeat_front_addr":"172.21.15.7:6831/472230900","state":["exists","up"]},{"osd":4,"uuid":"4da9bf66-cdfc-4620-96ac-848fd3a021a4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6800","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6801","nonce":3982341081}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6802","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6803","nonce":3982341081}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6806","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6807","nonce":3982341081}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6804","nonce":3982341081},{"type":"v1","addr":"172.21.15.194:6805","nonce":3982341081}]},"public_addr":"172.21.15.194:6801/3982341081","cluster_addr":"172.21.15.194:6803/3982341081","heartbeat_back_addr":"172.21.15.194:6807/3982341081","heartbeat_front_addr":"172.21.15.194:6805/3982341081","state":["exists","up"]},{"osd":5,"uuid":"bc079723-85f2-4e3d-9b95-35949f7bed75","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6808","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6809","nonce":636284694}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6810","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6811","nonce":636284694}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6814","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6815","nonce":636284694}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6812","nonce":636284694},{"type":"v1","addr":"172.21.15.194:6813","nonce":636284694}]},"public_addr":"172.21.15.194:6809/636284694","cluster_addr":"172.21.15.194:6811/636284694","heartbeat_back_addr":"172.21.15.194:6815/636284694","heartbeat_front_addr":"172.21.15.194:6813/636284694","state":["exists","up"]},{"osd":6,"uuid":"7a21b541-5d12-4cff-b2c1-990a294ce2d7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6816","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6817","nonce":435360745}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6818","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6819","nonce":435360745}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6822","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6823","nonce":435360745}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6820","nonce":435360745},{"type":"v1","addr":"172.21.15.194:6821","nonce":435360745}]},"public_addr":"172.21.15.194:6817/435360745","cluster_addr":"172.21.15.194:6819/435360745","heartbeat_back_addr":"172.21.15.194:6823/435360745","heartbeat_front_addr":"172.21.15.194:6821/435360745","state":["exists","up"]},{"osd":7,"uuid":"428284d0-2ee1-4431-ab27-0e5405f6e316","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6824","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6825","nonce":868693308}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6826","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6827","nonce":868693308}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6830","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6831","nonce":868693308}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.194:6828","nonce":868693308},{"type":"v1","addr":"172.21.15.194:6829","nonce":868693308}]},"public_addr":"172.21.15.194:6825/868693308","cluster_addr":"172.21.15.194:6827/868693308","heartbeat_back_addr":"172.21.15.194:6831/868693308","heartbeat_front_addr":"172.21.15.194:6829/868693308","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:53:18.086003+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:53:49.882579+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:54:16.844354+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:54:43.595494+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:04.781571+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:29.654845+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:55:53.380423+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-16T23:56:16.831974+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.7:6801/2587638492":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/310194880":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/1001861505":"2024-09-17T23:52:08.975108+0000","172.21.15.7:6800/2587638492":"2024-09-17T23:52:08.975108+0000","172.21.15.7:6801/2537923547":"2024-09-17T23:51:20.004938+0000","172.21.15.7:6800/2537923547":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/1404750921":"2024-09-17T23:52:08.975108+0000","172.21.15.7:0/955256528":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/3720859820":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/3879771779":"2024-09-17T23:50:51.103747+0000","172.21.15.7:6800/3263209100":"2024-09-17T23:50:51.103747+0000","172.21.15.7:6801/3263209100":"2024-09-17T23:50:51.103747+0000","172.21.15.7:0/2931702999":"2024-09-17T23:50:51.103747+0000","172.21.15.7:0/604101463":"2024-09-17T23:51:20.004938+0000","172.21.15.7:0/1304049360":"2024-09-17T23:50:51.103747+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-16T23:56:37.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:36 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/4181486143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T23:56:37.245 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:36 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/4181486143' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-16T23:56:37.270 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-16T23:56:37.271 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-16T23:56:37.271 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-16T23:56:37.272 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-16T23:56:37.272 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-16T23:56:37.272 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-16T23:56:37.273 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-16T23:56:37.273 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-16T23:56:37.711 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.731 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.731 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.742 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.742 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.764 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.771 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:37.777 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:38.102 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:37 smithi007 ceph-mon[23721]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:38.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:37 smithi194 ceph-mon[25713]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:40 smithi194 ceph-mon[25713]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:40.491 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:40 smithi007 ceph-mon[23721]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:42.238 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:42 smithi007 ceph-mon[23721]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:42.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:42 smithi194 ceph-mon[25713]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:44.197 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:44 smithi007 ceph-mon[23721]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:44 smithi194 ceph-mon[25713]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:46.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:45 smithi194 ceph-mon[25713]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:46.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:45 smithi007 ceph-mon[23721]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:47.627 INFO:teuthology.orchestra.run.smithi007.stdout:81604378656 2024-09-16T23:56:47.628 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-16T23:56:48.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:48 smithi007 ceph-mon[23721]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:48.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:48 smithi194 ceph-mon[25713]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:48.561 INFO:teuthology.orchestra.run.smithi007.stdout:60129542182 2024-09-16T23:56:48.561 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-16T23:56:50.212 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:50 smithi007 ceph-mon[23721]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:50.237 INFO:teuthology.orchestra.run.smithi007.stdout:167503724562 2024-09-16T23:56:50.237 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-16T23:56:50.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:50 smithi194 ceph-mon[25713]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:50.628 INFO:teuthology.orchestra.run.smithi007.stdout:111669149723 2024-09-16T23:56:50.628 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-16T23:56:51.209 INFO:teuthology.orchestra.run.smithi007.stdout:133143986200 2024-09-16T23:56:51.210 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-16T23:56:52.325 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:52 smithi007 ceph-mon[23721]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:52.354 INFO:teuthology.orchestra.run.smithi007.stdout:38654705709 2024-09-16T23:56:52.355 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-16T23:56:52.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:52 smithi194 ceph-mon[25713]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:53.120 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:53.120 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:53.121 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:53.765 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:54.041 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:54.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:54 smithi194 ceph-mon[25713]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:54.417 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:54 smithi007 ceph-mon[23721]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:55.334 INFO:teuthology.orchestra.run.smithi007.stdout:236223201290 2024-09-16T23:56:55.335 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-16T23:56:56.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:55 smithi194 ceph-mon[25713]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:56.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:55 smithi007 ceph-mon[23721]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:56.391 INFO:teuthology.orchestra.run.smithi007.stdout:201863462926 2024-09-16T23:56:56.392 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-16T23:56:58.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:58 smithi007 ceph-mon[23721]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:58.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:58 smithi194 ceph-mon[25713]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:56:58.433 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:58.434 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:56:59.232 INFO:teuthology.orchestra.run.smithi007.stdout:111669149725 2024-09-16T23:56:59.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:56:59 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2855084699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-16T23:56:59.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:56:59 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2855084699' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-16T23:56:59.848 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149723 got 111669149725 for osd.3 2024-09-16T23:56:59.849 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:00.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:00 smithi194 ceph-mon[25713]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:00.473 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:00 smithi007 ceph-mon[23721]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:02 smithi194 ceph-mon[25713]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:02.425 INFO:teuthology.orchestra.run.smithi007.stdout:81604378659 2024-09-16T23:57:02.425 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:02 smithi007 ceph-mon[23721]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:03.231 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:03 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/4121844112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-16T23:57:03.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:03 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/4121844112' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-16T23:57:03.539 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378656 got 81604378659 for osd.2 2024-09-16T23:57:03.539 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:04.258 INFO:teuthology.orchestra.run.smithi007.stdout:167503724566 2024-09-16T23:57:04.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:04 smithi007 ceph-mon[23721]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:04.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:04 smithi194 ceph-mon[25713]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:04.987 INFO:teuthology.orchestra.run.smithi007.stdout:60129542186 2024-09-16T23:57:05.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:05 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2164222311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-16T23:57:05.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:05 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/4029963674' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-16T23:57:05.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:05 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2164222311' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-16T23:57:05.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:05 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/4029963674' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-16T23:57:06.044 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:57:06.068 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724562 got 167503724566 for osd.5 2024-09-16T23:57:06.069 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:06.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:06 smithi007 ceph-mon[23721]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:06.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:06 smithi194 ceph-mon[25713]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:07.303 INFO:teuthology.orchestra.run.smithi007.stdout:236223201293 2024-09-16T23:57:07.535 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542182 got 60129542186 for osd.1 2024-09-16T23:57:07.536 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:07.907 INFO:teuthology.orchestra.run.smithi007.stdout:38654705713 2024-09-16T23:57:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:08 smithi194 ceph-mon[25713]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:08 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/871067943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-16T23:57:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:08 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1961293199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-16T23:57:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:08 smithi007 ceph-mon[23721]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:08 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/871067943' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-16T23:57:08.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:08 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1961293199' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-16T23:57:08.898 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201290 got 236223201293 for osd.7 2024-09-16T23:57:08.899 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:09.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:57:09.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:57:09.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:57:09.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:57:10.137 INFO:teuthology.orchestra.run.smithi007.stdout:133143986204 2024-09-16T23:57:10.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:10 smithi194 ceph-mon[25713]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:10.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:10 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2023597158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-16T23:57:10.522 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:10 smithi007 ceph-mon[23721]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:10.522 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:10 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2023597158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-16T23:57:10.676 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986200 got 133143986204 for osd.4 2024-09-16T23:57:10.676 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:11.063 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705713 for osd.0 2024-09-16T23:57:11.063 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:11.784 INFO:teuthology.orchestra.run.smithi007.stdout:201863462930 2024-09-16T23:57:12.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:12 smithi007 ceph-mon[23721]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:12.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:12 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2945835329' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-16T23:57:12.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:12 smithi194 ceph-mon[25713]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:12.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:12 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2945835329' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-16T23:57:12.457 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462926 got 201863462930 for osd.6 2024-09-16T23:57:12.457 DEBUG:teuthology.parallel:result is None 2024-09-16T23:57:12.457 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-16T23:57:12.457 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-16T23:57:12.679 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:57:14.316 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:14.318 INFO:teuthology.orchestra.run.smithi007.stderr:dumped all 2024-09-16T23:57:14.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:14 smithi007 ceph-mon[23721]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:14.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:14 smithi194 ceph-mon[25713]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:15.010 INFO:teuthology.orchestra.run.smithi007.stdout:{"pg_ready":true,"pg_map":{"version":199,"stamp":"2024-09-16T23:57:13.081501+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48988,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42624,"kb_avail":749912228,"statfs":{"total":767960285184,"available":767910121472,"internally_reserved":0,"allocated":6451200,"data_stored":3546640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43646976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002323"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-16T23:56:22.408664+0000","last_change":"2024-09-16T23:56:22.408664+0000","last_active":"2024-09-16T23:56:22.408664+0000","last_peered":"2024-09-16T23:56:22.408664+0000","last_clean":"2024-09-16T23:56:22.408664+0000","last_became_active":"2024-09-16T23:56:22.408200+0000","last_became_peered":"2024-09-16T23:56:22.408200+0000","last_unstale":"2024-09-16T23:56:22.408664+0000","last_undegraded":"2024-09-16T23:56:22.408664+0000","last_fullsized":"2024-09-16T23:56:22.408664+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-16T23:54:19.804337+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-16T23:54:19.804337+0000","last_clean_scrub_stamp":"2024-09-16T23:54:19.804337+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T08:54:46.843775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201294,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48299999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45300000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59799999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40500000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51000000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56200000000000006}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.57599999999999996}]},{"osd":2,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.61699999999999999}]},{"osd":3,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.60699999999999998}]},{"osd":4,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.52900000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.33900000000000002},"min":{"1min":0.27800000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.36399999999999999},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.70099999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Sep 16 23:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.41499999999999998,"15min":0.40699999999999997},"min":{"1min":0.33500000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.39800000000000002,"15min":0.39000000000000001},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Sep 16 23:56:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.45500000000000002,"15min":0.45100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.503}]},{"osd":4,"last update":"Mon Sep 16 23:56:08 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.70299999999999996}]},{"osd":5,"last update":"Mon Sep 16 23:56:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.628}]},{"osd":6,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.69699999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6948,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738204,"statfs":{"total":95995035648,"available":95987920896,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.53000000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.32400000000000001},"min":{"1min":0.249,"5min":0.219,"15min":0.219},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Sep 16 23:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.33000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.29799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.35699999999999998}]},{"osd":3,"last update":"Mon Sep 16 23:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.49199999999999999,"15min":0.496},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.57399999999999995,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Sep 16 23:56:09 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Sep 16 23:56:32 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.628},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.47999999999999998}]},{"osd":6,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.49099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":2,"up_from":19,"seq":81604378661,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.35599999999999998},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.38200000000000001,"15min":0.36299999999999999},"min":{"1min":0.28999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.255}]},{"osd":1,"last update":"Mon Sep 16 23:56:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.39200000000000002,"15min":0.38100000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.41299999999999998,"15min":0.40000000000000002},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.34499999999999997}]},{"osd":3,"last update":"Mon Sep 16 23:56:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.48599999999999999,"15min":0.47899999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.314},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.47199999999999998,"15min":0.47399999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.29999999999999999}]},{"osd":4,"last update":"Mon Sep 16 23:56:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.107,"5min":1.107,"15min":1.107},"last":0.55700000000000005}]},{"osd":5,"last update":"Mon Sep 16 23:56:35 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.53600000000000003}]},{"osd":6,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.436}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41899999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.39200000000000002,"15min":0.39000000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58599999999999997,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Sep 16 23:56:48 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.44400000000000001,"15min":0.45500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.60299999999999998,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.371},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.32400000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.307}]},{"osd":2,"last update":"Mon Sep 16 23:56:48 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47799999999999998,"15min":0.47999999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46400000000000002,"15min":0.46999999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.34100000000000003}]},{"osd":4,"last update":"Mon Sep 16 23:57:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.622,"15min":0.624},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.89600000000000002,"5min":1.01,"15min":1.01},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.62,"15min":0.61599999999999999},"min":{"1min":0.48599999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90300000000000002,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Sep 16 23:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.44600000000000001}]},{"osd":6,"last update":"Mon Sep 16 23:56:56 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.432},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.46400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77200000000000002,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.61699999999999999,"15min":0.61499999999999999},"min":{"1min":0.437,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79500000000000004,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.60499999999999998}]},{"osd":1,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56200000000000006,"15min":0.55700000000000005},"min":{"1min":0.40699999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.755,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.41899999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.67300000000000004,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.41899999999999998}]},{"osd":2,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.439,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.745,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.44800000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.69399999999999995,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.46800000000000003,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.745,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.45100000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.85999999999999999,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Sep 16 23:56:32 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.68600000000000005}]},{"osd":6,"last update":"Mon Sep 16 23:56:57 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.502}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.47899999999999998}]},{"osd":1,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61499999999999999}]},{"osd":3,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.66000000000000003}]},{"osd":4,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.53500000000000003}]},{"osd":6,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-16T23:57:15.012 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-16T23:57:15.229 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:57:15.497 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:15 smithi007 ceph-mon[23721]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:15.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:15 smithi194 ceph-mon[25713]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:16.317 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:16 smithi007 ceph-mon[23721]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:16.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:16 smithi194 ceph-mon[25713]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:17.005 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:17.008 INFO:teuthology.orchestra.run.smithi007.stderr:dumped all 2024-09-16T23:57:17.624 INFO:teuthology.orchestra.run.smithi007.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2024-09-16T23:57:15.081749+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48988,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":42624,"kb_avail":749912228,"statfs":{"total":767960285184,"available":767910121472,"internally_reserved":0,"allocated":6451200,"data_stored":3546640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43646976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002088"},"pg_stats":[{"pgid":"1.0","version":"22'117","reported_seq":226,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-16T23:56:22.408664+0000","last_change":"2024-09-16T23:56:22.408664+0000","last_active":"2024-09-16T23:56:22.408664+0000","last_peered":"2024-09-16T23:56:22.408664+0000","last_clean":"2024-09-16T23:56:22.408664+0000","last_became_active":"2024-09-16T23:56:22.408200+0000","last_became_peered":"2024-09-16T23:56:22.408200+0000","last_unstale":"2024-09-16T23:56:22.408664+0000","last_undegraded":"2024-09-16T23:56:22.408664+0000","last_fullsized":"2024-09-16T23:56:22.408664+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-16T23:54:19.804337+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-16T23:54:19.804337+0000","last_clean_scrub_stamp":"2024-09-16T23:54:19.804337+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-18T08:54:46.843775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6048,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739104,"statfs":{"total":95995035648,"available":95988842496,"internally_reserved":0,"allocated":1138688,"data_stored":773838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66100000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72899999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50800000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462930,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6172,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93738980,"statfs":{"total":95995035648,"available":95988715520,"internally_reserved":0,"allocated":1134592,"data_stored":773594,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.58599999999999997}]},{"osd":1,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.57599999999999996}]},{"osd":2,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.61699999999999999}]},{"osd":3,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.60699999999999998}]},{"osd":4,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46400000000000002}]},{"osd":5,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.52900000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]}]},{"osd":1,"up_from":14,"seq":60129542188,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6360,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738792,"statfs":{"total":95995035648,"available":95988523008,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.33900000000000002},"min":{"1min":0.27800000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.39200000000000002,"15min":0.36399999999999999},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.70099999999999996,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.47699999999999998}]},{"osd":2,"last update":"Mon Sep 16 23:56:25 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.41499999999999998,"15min":0.40699999999999997},"min":{"1min":0.33500000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.39800000000000002,"15min":0.39000000000000001},"min":{"1min":0.309,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.42099999999999999}]},{"osd":3,"last update":"Mon Sep 16 23:56:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.45500000000000002,"15min":0.45100000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.503}]},{"osd":4,"last update":"Mon Sep 16 23:56:08 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.70299999999999996}]},{"osd":5,"last update":"Mon Sep 16 23:56:33 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.628}]},{"osd":6,"last update":"Mon Sep 16 23:56:55 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.69699999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56799999999999995}]}]},{"osd":0,"up_from":9,"seq":38654705714,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6948,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738204,"statfs":{"total":95995035648,"available":95987920896,"internally_reserved":0,"allocated":1142784,"data_stored":774161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.53000000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35699999999999998,"15min":0.32400000000000001},"min":{"1min":0.249,"5min":0.219,"15min":0.219},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.36699999999999999}]},{"osd":2,"last update":"Mon Sep 16 23:56:27 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.40799999999999997,"15min":0.40200000000000002},"min":{"1min":0.33000000000000002,"5min":0.246,"15min":0.246},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.38700000000000001,"15min":0.375},"min":{"1min":0.29799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.35699999999999998}]},{"osd":3,"last update":"Mon Sep 16 23:56:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.49199999999999999,"15min":0.496},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.46300000000000002,"15min":0.46500000000000002},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.57399999999999995,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.38500000000000001}]},{"osd":4,"last update":"Mon Sep 16 23:56:09 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58499999999999996}]},{"osd":5,"last update":"Mon Sep 16 23:56:32 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.628},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.47999999999999998}]},{"osd":6,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.49099999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":2,"up_from":19,"seq":81604378662,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6300,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738852,"statfs":{"total":95995035648,"available":95988584448,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:26 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.36499999999999999,"15min":0.35599999999999998},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.495,"5min":0.38200000000000001,"15min":0.36299999999999999},"min":{"1min":0.28999999999999998,"5min":0.254,"15min":0.254},"max":{"1min":1.1619999999999999,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.433}]},{"osd":1,"last update":"Mon Sep 16 23:56:26 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.39200000000000002,"15min":0.38100000000000001},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.308},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.41299999999999998,"15min":0.40000000000000002},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.40799999999999997}]},{"osd":3,"last update":"Mon Sep 16 23:56:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.48599999999999999,"15min":0.47899999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.47199999999999998,"15min":0.47399999999999998},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.375}]},{"osd":4,"last update":"Mon Sep 16 23:57:11 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.59699999999999998,"15min":0.59599999999999997},"min":{"1min":0.44700000000000001,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.86499999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.65200000000000002,"15min":0.65400000000000003},"min":{"1min":0.48299999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.90400000000000003,"5min":1.107,"15min":1.107},"last":0.54500000000000004}]},{"osd":5,"last update":"Mon Sep 16 23:56:35 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.0489999999999999,"5min":1.0489999999999999,"15min":1.0489999999999999},"last":0.498},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.59899999999999998}]},{"osd":6,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52300000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149728,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5720,"kb_used_data":592,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739432,"statfs":{"total":95995035648,"available":95989178368,"internally_reserved":0,"allocated":606208,"data_stored":244968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.39200000000000002,"15min":0.39000000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.44800000000000001,"15min":0.45200000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.58599999999999997,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.29099999999999998}]},{"osd":1,"last update":"Mon Sep 16 23:56:48 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.44400000000000001,"15min":0.45500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.60299999999999998,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.371},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.45600000000000002,"15min":0.45800000000000002},"min":{"1min":0.32400000000000001,"5min":0.316,"15min":0.316},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.307}]},{"osd":2,"last update":"Mon Sep 16 23:56:48 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47799999999999998,"15min":0.47999999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.46400000000000002,"15min":0.46999999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.34100000000000003}]},{"osd":4,"last update":"Mon Sep 16 23:57:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.622,"15min":0.624},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.89600000000000002,"5min":1.01,"15min":1.01},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.62,"15min":0.61599999999999999},"min":{"1min":0.48599999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.90300000000000002,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.54200000000000004}]},{"osd":5,"last update":"Mon Sep 16 23:56:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.0620000000000001,"5min":1.0620000000000001,"15min":1.0620000000000001},"last":0.44600000000000001}]},{"osd":6,"last update":"Mon Sep 16 23:56:56 2024","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.503,"5min":0.503,"15min":0.503},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.432},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.46400000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986205,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5724,"kb_used_data":596,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739428,"statfs":{"total":95995035648,"available":95989174272,"internally_reserved":0,"allocated":610304,"data_stored":245233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59899999999999998,"15min":0.59999999999999998},"min":{"1min":0.39900000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.77200000000000002,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.61699999999999999,"15min":0.61499999999999999},"min":{"1min":0.437,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79500000000000004,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.60499999999999998}]},{"osd":1,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.56200000000000006,"15min":0.55700000000000005},"min":{"1min":0.40699999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.755,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56100000000000005,"15min":0.55900000000000005},"min":{"1min":0.41899999999999998,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.67300000000000004,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.41899999999999998}]},{"osd":2,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.57299999999999995,"15min":0.56799999999999995},"min":{"1min":0.439,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.745,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.44800000000000001,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.69399999999999995,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.56200000000000006}]},{"osd":3,"last update":"Mon Sep 16 23:57:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.59299999999999997,"15min":0.59399999999999997},"min":{"1min":0.46800000000000003,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.745,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.45100000000000001,"5min":0.375,"15min":0.375},"max":{"1min":0.85999999999999999,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.53600000000000003}]},{"osd":5,"last update":"Mon Sep 16 23:56:32 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.68600000000000005}]},{"osd":6,"last update":"Mon Sep 16 23:56:57 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.502}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5716,"kb_used_data":588,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739436,"statfs":{"total":95995035648,"available":95989182464,"internally_reserved":0,"allocated":602112,"data_stored":244645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.47899999999999998}]},{"osd":1,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.52400000000000002}]},{"osd":2,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.61499999999999999}]},{"osd":3,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.66000000000000003}]},{"osd":4,"last update":"Mon Sep 16 23:56:34 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.53500000000000003}]},{"osd":6,"last update":"Mon Sep 16 23:56:58 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58399999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-16T23:57:17.626 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-16T23:57:17.626 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-16T23:57:17.626 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-16T23:57:17.626 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-16T23:57:17.846 INFO:teuthology.orchestra.run.smithi007.stderr:Inferring config /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/mon.smithi007/config 2024-09-16T23:57:18.284 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:18 smithi007 ceph-mon[23721]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:18.284 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:18 smithi007 ceph-mon[23721]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:18.452 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:18 smithi194 ceph-mon[25713]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:18.452 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:18 smithi194 ceph-mon[25713]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:19.618 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:19.618 INFO:teuthology.orchestra.run.smithi007.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-16T23:57:20.277 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:20 smithi007 ceph-mon[23721]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:20.277 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:20 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/591016994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-16T23:57:20.305 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-16T23:57:20.305 INFO:tasks.cephadm:Setup complete, yielding 2024-09-16T23:57:20.305 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T23:57:20.318 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:57:20.318 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-16T23:57:20.436 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:20 smithi194 ceph-mon[25713]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:20.436 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:20 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/591016994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-16T23:57:21.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:21.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:21.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:21.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:21 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:21.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:21.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:21.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:21.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:21 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:22.303 INFO:teuthology.orchestra.run.smithi007.stdout:Backend: cephadm 2024-09-16T23:57:22.303 INFO:teuthology.orchestra.run.smithi007.stdout:Available: Yes 2024-09-16T23:57:22.303 INFO:teuthology.orchestra.run.smithi007.stdout:Paused: No 2024-09-16T23:57:22.574 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:22 smithi007 ceph-mon[23721]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:22.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:22 smithi194 ceph-mon[25713]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:22.931 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-16T23:57:23.503 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:23 smithi007 ceph-mon[23721]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:23.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:23 smithi194 ceph-mon[25713]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:24.457 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:24 smithi007 ceph-mon[23721]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:24 smithi194 ceph-mon[25713]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:24.904 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T23:57:24.904 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (4m) 2m ago 5m 16.7M - ba2b418f427c ddb6debdf7d2 2024-09-16T23:57:24.904 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (5m) 2m ago 5m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-16T23:57:24.904 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (5m) 67s ago 5m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (4m) 2m ago 5m 45.4M - 8.3.5 dad864ee21e9 de7bd16dee04 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (6m) 2m ago 6m 448M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (5m) 67s ago 5m 413M - 17.2.0 e1d6a67b021e 1223cf6c34e2 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (6m) 2m ago 7m 42.1M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (4m) 67s ago 4m 39.1M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (5m) 2m ago 5m 17.9M - 1dbe0e931976 59ef89f4df45 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (4m) 67s ago 4m 16.8M - 1dbe0e931976 1631f9c583ae 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (4m) 2m ago 4m 35.1M 3217M 17.2.0 e1d6a67b021e 78961710495e 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (3m) 2m ago 3m 34.3M 3217M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (3m) 2m ago 3m 30.0M 3217M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (2m) 2m ago 2m 10.9M 3217M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (2m) 67s ago 2m 32.8M 3984M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-16T23:57:24.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (119s) 67s ago 119s 33.3M 3984M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-16T23:57:24.906 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (96s) 67s ago 95s 33.1M 3984M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-16T23:57:24.906 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (72s) 67s ago 72s 10.6M 3984M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-16T23:57:24.906 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (4m) 2m ago 4m 53.6M - 514e6a882f6e c6f8e3619d7a 2024-09-16T23:57:25.533 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-16T23:57:26.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:25 smithi007 ceph-mon[23721]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:26.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:25 smithi007 ceph-mon[23721]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:26.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:25 smithi194 ceph-mon[25713]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:25 smithi194 ceph-mon[25713]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:27.480 INFO:teuthology.orchestra.run.smithi007.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager ?:9093,9094 1/1 2m ago 6m count:1 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:crash 2/2 2m ago 6m * 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:grafana ?:3000 1/1 2m ago 6m count:1 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:mgr 2/2 2m ago 6m count:2 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:mon 2/2 2m ago 5m smithi007:172.21.15.7=smithi007;smithi194:172.21.15.194=smithi194;count:2 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter ?:9100 2/2 2m ago 6m * 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:osd 8 2m ago - 2024-09-16T23:57:27.481 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus ?:9095 1/1 2m ago 6m count:1 2024-09-16T23:57:28.119 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-16T23:57:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:28 smithi007 ceph-mon[23721]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:28.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:28 smithi007 ceph-mon[23721]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:28 smithi194 ceph-mon[25713]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:28.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:28 smithi194 ceph-mon[25713]: from='client.14456 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:30.094 INFO:teuthology.orchestra.run.smithi007.stdout:HOST ADDR LABELS STATUS 2024-09-16T23:57:30.094 INFO:teuthology.orchestra.run.smithi007.stdout:smithi007 172.21.15.7 2024-09-16T23:57:30.094 INFO:teuthology.orchestra.run.smithi007.stdout:smithi194 172.21.15.194 2024-09-16T23:57:30.094 INFO:teuthology.orchestra.run.smithi007.stdout:2 hosts in cluster 2024-09-16T23:57:30.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:30 smithi007 ceph-mon[23721]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:30.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:30 smithi194 ceph-mon[25713]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:30.675 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-16T23:57:31.292 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:31 smithi007 ceph-mon[23721]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:31.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:31 smithi194 ceph-mon[25713]: from='client.14460 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:32.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:32 smithi194 ceph-mon[25713]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:32.553 INFO:teuthology.orchestra.run.smithi007.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-16T23:57:32.553 INFO:teuthology.orchestra.run.smithi007.stdout:smithi007 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB751300GE480DGN 480G LVM detected, locked 2024-09-16T23:57:32.553 INFO:teuthology.orchestra.run.smithi007.stdout:smithi194 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6240002A400BGN 400G LVM detected, locked 2024-09-16T23:57:32.554 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:32 smithi007 ceph-mon[23721]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:33.203 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:33 smithi007 ceph-mon[23721]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:33.227 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T23:57:33.240 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:57:33.240 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-16T23:57:33.267 INFO:teuthology.orchestra.run.smithi007.stderr:+ systemctl stop nfs-server 2024-09-16T23:57:33.275 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi194.front.sepia.ceph.com 2024-09-16T23:57:33.275 DEBUG:teuthology.orchestra.run.smithi194:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-16T23:57:33.307 INFO:teuthology.orchestra.run.smithi194.stderr:+ systemctl stop nfs-server 2024-09-16T23:57:33.315 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T23:57:33.325 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:57:33.325 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-16T23:57:33.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:33 smithi194 ceph-mon[25713]: from='client.14464 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:34.292 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:34 smithi007 ceph-mon[23721]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:34.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:34 smithi194 ceph-mon[25713]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:36.235 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:35 smithi007 ceph-mon[23721]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:36.236 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:35 smithi007 ceph-mon[23721]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:36.236 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:35 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-16T23:57:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:35 smithi194 ceph-mon[25713]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:35 smithi194 ceph-mon[25713]: from='client.14468 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:35 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-16T23:57:37.301 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:36 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-16T23:57:37.301 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:36 smithi007 ceph-mon[23721]: osdmap e60: 8 total, 8 up, 8 in 2024-09-16T23:57:37.301 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:36 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-16T23:57:37.301 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:36 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[23717]: 2024-09-16T23:57:36.930+0000 7fcb9ee75700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T23:57:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:36 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-16T23:57:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:36 smithi194 ceph-mon[25713]: osdmap e60: 8 total, 8 up, 8 in 2024-09-16T23:57:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:36 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-16T23:57:37.711 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-16T23:57:37.722 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-16T23:57:37.722 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T23:57:38.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-16T23:57:38.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: osdmap e61: 8 total, 8 up, 8 in 2024-09-16T23:57:38.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-16T23:57:38.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T23:57:38.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-16T23:57:38.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: osdmap e62: 8 total, 8 up, 8 in 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: fsmap foofs:0 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: Saving service mds.foofs spec with placement count:2 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:38.045 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: Deploying daemon mds.foofs.smithi007.fkkfbl on smithi007 2024-09-16T23:57:38.046 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:37 smithi007 ceph-mon[23721]: pgmap v214: 65 pgs: 5 creating+peering, 1 active+clean, 59 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: osdmap e61: 8 total, 8 up, 8 in 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: osdmap e62: 8 total, 8 up, 8 in 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-16T23:57:38.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: fsmap foofs:0 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: Saving service mds.foofs spec with placement count:2 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: Deploying daemon mds.foofs.smithi007.fkkfbl on smithi007 2024-09-16T23:57:38.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:37 smithi194 ceph-mon[25713]: pgmap v214: 65 pgs: 5 creating+peering, 1 active+clean, 59 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:39.303 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:38 smithi007 ceph-mon[23721]: osdmap e63: 8 total, 8 up, 8 in 2024-09-16T23:57:39.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:38 smithi194 ceph-mon[25713]: osdmap e63: 8 total, 8 up, 8 in 2024-09-16T23:57:40.249 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:39 smithi007 ceph-mon[23721]: osdmap e64: 8 total, 8 up, 8 in 2024-09-16T23:57:40.249 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:39 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:40.249 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:39 smithi007 ceph-mon[23721]: pgmap v217: 65 pgs: 17 creating+peering, 25 active+clean, 23 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:39 smithi194 ceph-mon[25713]: osdmap e64: 8 total, 8 up, 8 in 2024-09-16T23:57:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:39 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:39 smithi194 ceph-mon[25713]: pgmap v217: 65 pgs: 17 creating+peering, 25 active+clean, 23 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:40.996 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:40.997 INFO:teuthology.orchestra.run.smithi007.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T23:51:09.466050Z", "last_refresh": "2024-09-16T23:54:46.206560Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T23:51:03.946294Z", "last_refresh": "2024-09-16T23:54:46.206628Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T23:51:06.592239Z", "last_refresh": "2024-09-16T23:54:46.206695Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T23:57:36.951669Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T23:57:36.943452Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T23:51:01.993043Z", "last_refresh": "2024-09-16T23:54:46.206485Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi007:172.21.15.7=smithi007", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T23:51:53.906495Z", "last_refresh": "2024-09-16T23:54:46.206344Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T23:51:08.211904Z", "last_refresh": "2024-09-16T23:54:46.206759Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T23:54:46.206900Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T23:51:05.157862Z", "last_refresh": "2024-09-16T23:54:46.206835Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T23:57:41.861 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-16T23:57:42.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:42.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: pgmap v218: 65 pgs: 17 creating+peering, 36 active+clean, 12 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: Deploying daemon mds.foofs.smithi194.emredf on smithi194 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: mds.? [v2:172.21.15.7:6834/4061963992,v1:172.21.15.7:6835/4061963992] up:boot 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: daemon mds.foofs.smithi007.fkkfbl assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: Cluster is now healthy 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: fsmap foofs:0 1 up:standby 2024-09-16T23:57:42.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-16T23:57:42.851 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:creating} 2024-09-16T23:57:42.851 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:42 smithi007 ceph-mon[23721]: daemon mds.foofs.smithi007.fkkfbl is now active in filesystem foofs as rank 0 2024-09-16T23:57:42.862 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: from='client.14472 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: pgmap v218: 65 pgs: 17 creating+peering, 36 active+clean, 12 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: Deploying daemon mds.foofs.smithi194.emredf on smithi194 2024-09-16T23:57:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: mds.? [v2:172.21.15.7:6834/4061963992,v1:172.21.15.7:6835/4061963992] up:boot 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: daemon mds.foofs.smithi007.fkkfbl assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: Cluster is now healthy 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: fsmap foofs:0 1 up:standby 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:creating} 2024-09-16T23:57:42.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:42 smithi194 ceph-mon[25713]: daemon mds.foofs.smithi007.fkkfbl is now active in filesystem foofs as rank 0 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 499 B/s wr, 0 op/s 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: mds.? [v2:172.21.15.7:6834/4061963992,v1:172.21.15.7:6835/4061963992] up:active 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:44 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: pgmap v219: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 499 B/s wr, 0 op/s 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: mds.? [v2:172.21.15.7:6834/4061963992,v1:172.21.15.7:6835/4061963992] up:active 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:44 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:44.849 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:44.849 INFO:teuthology.orchestra.run.smithi007.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T23:51:09.466050Z", "last_refresh": "2024-09-16T23:54:46.206560Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T23:51:03.946294Z", "last_refresh": "2024-09-16T23:54:46.206628Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T23:51:06.592239Z", "last_refresh": "2024-09-16T23:54:46.206695Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T23:57:36.951669Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T23:57:36.943452Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T23:51:01.993043Z", "last_refresh": "2024-09-16T23:54:46.206485Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi007:172.21.15.7=smithi007", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T23:51:53.906495Z", "last_refresh": "2024-09-16T23:54:46.206344Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T23:51:08.211904Z", "last_refresh": "2024-09-16T23:54:46.206759Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T23:54:46.206900Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T23:51:05.157862Z", "last_refresh": "2024-09-16T23:54:46.206835Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T23:57:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:45 smithi007 ceph-mon[23721]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:boot 2024-09-16T23:57:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:45 smithi007 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-16T23:57:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:45 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-16T23:57:45.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:45 smithi007 ceph-mon[23721]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-16T23:57:45.695 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-16T23:57:45.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:45 smithi194 ceph-mon[25713]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:boot 2024-09-16T23:57:45.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:45 smithi194 ceph-mon[25713]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-16T23:57:45.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:45 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-16T23:57:45.845 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:45 smithi194 ceph-mon[25713]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-16T23:57:46.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:46 smithi007 ceph-mon[23721]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:46.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:46 smithi007 ceph-mon[23721]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 383 B/s wr, 3 op/s 2024-09-16T23:57:46.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:46 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:46.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:46 smithi194 ceph-mon[25713]: from='client.14482 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:46 smithi194 ceph-mon[25713]: pgmap v220: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 383 B/s wr, 3 op/s 2024-09-16T23:57:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:46 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:46.696 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T23:57:48.674 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:48 smithi007 ceph-mon[23721]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-16T23:57:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:48 smithi194 ceph-mon[25713]: pgmap v221: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-16T23:57:50.021 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:50.021 INFO:teuthology.orchestra.run.smithi007.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T23:51:09.466050Z", "last_refresh": "2024-09-16T23:54:46.206560Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T23:51:03.946294Z", "last_refresh": "2024-09-16T23:54:46.206628Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T23:51:06.592239Z", "last_refresh": "2024-09-16T23:54:46.206695Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T23:57:36.951669Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T23:57:36.943452Z", "last_refresh": "2024-09-16T23:57:45.852346Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T23:51:01.993043Z", "last_refresh": "2024-09-16T23:54:46.206485Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi007:172.21.15.7=smithi007", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T23:51:53.906495Z", "last_refresh": "2024-09-16T23:54:46.206344Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T23:51:08.211904Z", "last_refresh": "2024-09-16T23:54:46.206759Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T23:54:46.206900Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T23:51:05.157862Z", "last_refresh": "2024-09-16T23:54:46.206835Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T23:57:50.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:50 smithi194 ceph-mon[25713]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-16T23:57:50.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:50 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:50.493 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:50 smithi007 ceph-mon[23721]: pgmap v222: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-16T23:57:50.494 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:50 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:50.878 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-16T23:57:51.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:51 smithi194 ceph-mon[25713]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:51.492 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:51 smithi007 ceph-mon[23721]: from='client.14486 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:51.879 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:52 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: pgmap v223: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:52 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:53.969 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:57:53.970 INFO:teuthology.orchestra.run.smithi007.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T23:51:09.466050Z", "last_refresh": "2024-09-16T23:57:51.498877Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T23:51:03.946294Z", "last_refresh": "2024-09-16T23:57:45.851609Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-16T23:51:06.592239Z", "last_refresh": "2024-09-16T23:57:51.499011Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-16T23:57:36.951669Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T23:57:36.943452Z", "last_refresh": "2024-09-16T23:57:45.852346Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T23:51:01.993043Z", "last_refresh": "2024-09-16T23:57:45.851779Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi007:172.21.15.7=smithi007", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T23:51:53.906495Z", "last_refresh": "2024-09-16T23:57:45.851891Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T23:51:08.211904Z", "last_refresh": "2024-09-16T23:57:45.851972Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-16T23:57:45.852049Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T23:51:05.157862Z", "last_refresh": "2024-09-16T23:57:51.499138Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-16T23:57:54.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:54 smithi007 ceph-mon[23721]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T23:57:54.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:54 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:54.679 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-16T23:57:54.679 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-16T23:57:54.688 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:57:54.689 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-16T23:57:54.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:54 smithi194 ceph-mon[25713]: pgmap v224: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-16T23:57:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:54 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:55.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:55 smithi007 ceph-mon[23721]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:55 smithi194 ceph-mon[25713]: from='client.14490 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-16T23:57:56.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:56 smithi007 ceph-mon[23721]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 3 op/s 2024-09-16T23:57:56.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:56 smithi194 ceph-mon[25713]: pgmap v225: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 3 op/s 2024-09-16T23:57:57.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:57 smithi007 ceph-mon[23721]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:57.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:57 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-16T23:57:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:57 smithi194 ceph-mon[25713]: from='client.14494 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:57:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:57 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-16T23:57:58.549 INFO:teuthology.orchestra.run.smithi007.stdout:NFS Cluster Created Successfully 2024-09-16T23:57:58.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:58 smithi007 ceph-mon[23721]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 1 op/s 2024-09-16T23:57:58.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:58 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-16T23:57:58.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:58 smithi007 ceph-mon[23721]: osdmap e65: 8 total, 8 up, 8 in 2024-09-16T23:57:58.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:58 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-16T23:57:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:58 smithi194 ceph-mon[25713]: pgmap v226: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 853 B/s wr, 1 op/s 2024-09-16T23:57:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:58 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-16T23:57:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:58 smithi194 ceph-mon[25713]: osdmap e65: 8 total, 8 up, 8 in 2024-09-16T23:57:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:58 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-16T23:57:59.313 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-16T23:57:59.802 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-16T23:57:59.802 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: osdmap e66: 8 total, 8 up, 8 in 2024-09-16T23:57:59.802 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: Saving service nfs.foo spec with placement count:2 2024-09-16T23:57:59.802 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:59.802 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: Creating key for client.nfs.foo.0.0.smithi007.lardmp 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T23:57:59.803 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T23:57:59.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T23:57:59.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T23:57:59.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:57:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: osdmap e66: 8 total, 8 up, 8 in 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: Saving service nfs.foo spec with placement count:2 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:57:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: Creating key for client.nfs.foo.0.0.smithi007.lardmp 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T23:57:59.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:57:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:00.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:00 smithi194 ceph-mon[25713]: Rados config object exists: conf-nfs.foo 2024-09-16T23:58:00.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:00 smithi194 ceph-mon[25713]: Creating key for client.nfs.foo.0.0.smithi007.lardmp-rgw 2024-09-16T23:58:00.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:00 smithi194 ceph-mon[25713]: Deploying daemon nfs.foo.0.0.smithi007.lardmp on smithi007 2024-09-16T23:58:00.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:00 smithi194 ceph-mon[25713]: pgmap v229: 97 pgs: 11 creating+peering, 21 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:58:00.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:00 smithi194 ceph-mon[25713]: osdmap e67: 8 total, 8 up, 8 in 2024-09-16T23:58:00.810 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:00 smithi194 ceph-mon[25713]: mgrmap e20: smithi007.czdphd(active, since 5m), standbys: smithi194.xlteft 2024-09-16T23:58:00.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:00 smithi007 ceph-mon[23721]: Rados config object exists: conf-nfs.foo 2024-09-16T23:58:00.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:00 smithi007 ceph-mon[23721]: Creating key for client.nfs.foo.0.0.smithi007.lardmp-rgw 2024-09-16T23:58:00.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:00 smithi007 ceph-mon[23721]: Deploying daemon nfs.foo.0.0.smithi007.lardmp on smithi007 2024-09-16T23:58:00.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:00 smithi007 ceph-mon[23721]: pgmap v229: 97 pgs: 11 creating+peering, 21 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:58:00.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:00 smithi007 ceph-mon[23721]: osdmap e67: 8 total, 8 up, 8 in 2024-09-16T23:58:00.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:00 smithi007 ceph-mon[23721]: mgrmap e20: smithi007.czdphd(active, since 5m), standbys: smithi194.xlteft 2024-09-16T23:58:02.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:02 smithi007 ceph-mon[23721]: pgmap v231: 97 pgs: 11 creating+peering, 17 unknown, 69 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:58:02.779 INFO:teuthology.orchestra.run.smithi007.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-16T23:58:02.779 INFO:teuthology.orchestra.run.smithi007.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-16T23:58:02.779 INFO:teuthology.orchestra.run.smithi007.stderr:Error EINVAL: invalid command 2024-09-16T23:58:02.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:02 smithi194 ceph-mon[25713]: pgmap v231: 97 pgs: 11 creating+peering, 17 unknown, 69 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-16T23:58:03.148 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:58:03.148 INFO:teuthology.orchestra.run.smithi007.stdout: "bind": "/fake", 2024-09-16T23:58:03.148 INFO:teuthology.orchestra.run.smithi007.stdout: "fs": "foofs", 2024-09-16T23:58:03.148 INFO:teuthology.orchestra.run.smithi007.stdout: "path": "/", 2024-09-16T23:58:03.148 INFO:teuthology.orchestra.run.smithi007.stdout: "cluster": "foo", 2024-09-16T23:58:03.149 INFO:teuthology.orchestra.run.smithi007.stdout: "mode": "RW" 2024-09-16T23:58:03.149 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:58:03.856 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T23:58:04.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T23:58:04.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:04.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-16T23:58:04.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:04.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-16T23:58:04.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:03 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-16T23:58:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:04.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: pgmap v232: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 341 B/s wr, 0 op/s 2024-09-16T23:58:04.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='client.14526 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:04.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-16T23:58:04.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:03 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-16T23:58:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:05 smithi007 ceph-mon[23721]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 135 B/s rd, 270 B/s wr, 0 op/s 2024-09-16T23:58:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:05 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T23:58:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:05 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T23:58:06.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:05 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T23:58:06.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:05 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T23:58:06.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:05 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:05 smithi194 ceph-mon[25713]: pgmap v233: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 135 B/s rd, 270 B/s wr, 0 op/s 2024-09-16T23:58:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:05 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-16T23:58:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:05 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-16T23:58:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:05 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-16T23:58:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:05 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-16T23:58:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:05 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:07.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:06 smithi194 ceph-mon[25713]: Rados config object exists: conf-nfs.foo 2024-09-16T23:58:07.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:06 smithi194 ceph-mon[25713]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi-rgw 2024-09-16T23:58:07.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:06 smithi194 ceph-mon[25713]: Deploying daemon nfs.foo.1.0.smithi194.wpbzoi on smithi194 2024-09-16T23:58:07.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:06 smithi194 ceph-mon[25713]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:06 smithi007 ceph-mon[23721]: Rados config object exists: conf-nfs.foo 2024-09-16T23:58:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:06 smithi007 ceph-mon[23721]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi-rgw 2024-09-16T23:58:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:06 smithi007 ceph-mon[23721]: Deploying daemon nfs.foo.1.0.smithi194.wpbzoi on smithi194 2024-09-16T23:58:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:06 smithi007 ceph-mon[23721]: from='client.14540 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:08.188 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:07 smithi194 ceph-mon[25713]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-16T23:58:08.189 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:07 smithi194 ceph-mon[25713]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:08.192 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:07 smithi007 ceph-mon[23721]: pgmap v234: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1.3 KiB/s wr, 3 op/s 2024-09-16T23:58:08.192 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:07 smithi007 ceph-mon[23721]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:09.453 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:09.453 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:58:09.454 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:09.480 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:09.480 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:58:09.480 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:09.480 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:58:09.481 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]': finished 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: osdmap e68: 8 total, 8 up, 8 in 2024-09-16T23:58:10.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:10 smithi007 ceph-mon[23721]: osdmap e69: 8 total, 8 up, 8 in 2024-09-16T23:58:10.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: from='client.14554 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: pgmap v235: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1.1 KiB/s wr, 2 op/s 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1d", "id": [7, 2]}]': finished 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: osdmap e68: 8 total, 8 up, 8 in 2024-09-16T23:58:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:10 smithi194 ceph-mon[25713]: osdmap e69: 8 total, 8 up, 8 in 2024-09-16T23:58:11.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:11 smithi194 ceph-mon[25713]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:11.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:11 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:11.958 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:11 smithi007 ceph-mon[23721]: from='client.14558 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:11.959 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:11 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:13.002 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:12 smithi007 ceph-mon[23721]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.1 KiB/s wr, 4 op/s 2024-09-16T23:58:13.002 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:12 smithi007 ceph-mon[23721]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:12 smithi194 ceph-mon[25713]: pgmap v238: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 2.1 KiB/s wr, 4 op/s 2024-09-16T23:58:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:12 smithi194 ceph-mon[25713]: from='client.14562 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:14.057 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo ?:2049 2/2 3s ago 15s count:2 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:14.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:58:14.058 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.058 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:13 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='client.14566 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:13 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:14.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:14 smithi007 ceph-mon[23721]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-16T23:58:14.887 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-16T23:58:14.897 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:58:14.897 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-16T23:58:14.926 INFO:teuthology.orchestra.run.smithi007.stderr:+ mkdir /mnt/foo 2024-09-16T23:58:14.937 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-16T23:58:15.005 INFO:teuthology.orchestra.run.smithi007.stderr:++ hostname 2024-09-16T23:58:15.005 INFO:teuthology.orchestra.run.smithi007.stderr:+ mount -t nfs smithi007:/fake /mnt/foo -o sync 2024-09-16T23:58:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:14 smithi194 ceph-mon[25713]: pgmap v239: 97 pgs: 2 peering, 95 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.1 KiB/s wr, 8 op/s 2024-09-16T23:58:15.496 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-16T23:58:15.562 INFO:teuthology.orchestra.run.smithi007.stderr:+ echo test 2024-09-16T23:58:15.573 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-16T23:58:15.638 INFO:teuthology.orchestra.run.smithi007.stderr:+ sync 2024-09-16T23:58:15.848 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:15 smithi007 ceph-mon[23721]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:16.063 INFO:teuthology.run_tasks:Running task parallel... 2024-09-16T23:58:16.074 INFO:teuthology.task.parallel:starting parallel... 2024-09-16T23:58:16.074 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-16T23:58:16.075 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-16T23:58:16.075 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:58:16.075 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-16T23:58:16.076 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-16T23:58:16.076 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-16T23:58:16.077 INFO:teuthology.task.exec:Executing custom commands... 2024-09-16T23:58:16.077 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:58:16.077 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-16T23:58:16.137 INFO:teuthology.orchestra.run.smithi007.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-16T23:58:16.138 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-16T23:58:16.138 INFO:teuthology.orchestra.run.smithi007.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-16T23:58:16.139 INFO:teuthology.orchestra.run.smithi007.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-16T23:58:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:15 smithi194 ceph-mon[25713]: from='client.14570 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:16.239 INFO:teuthology.orchestra.run.smithi007.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-16T23:58:16.239 INFO:teuthology.orchestra.run.smithi007.stdout:releasing clients 2024-09-16T23:58:16.889 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:16 smithi007 ceph-mon[23721]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 KiB/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:17.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:16 smithi194 ceph-mon[25713]: pgmap v240: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.0 KiB/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:17.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 92 20.97 MB/sec warmup 1 sec latency 31.670 ms 2024-09-16T23:58:18.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 195 19.35 MB/sec warmup 2 sec latency 39.494 ms 2024-09-16T23:58:18.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:18 smithi007 ceph-mon[23721]: pgmap v241: 97 pgs: 97 active+clean; 2.9 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 308 KiB/s wr, 11 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:19.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:18 smithi194 ceph-mon[25713]: pgmap v241: 97 pgs: 97 active+clean; 2.9 MiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 308 KiB/s wr, 11 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:19.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 285 17.65 MB/sec warmup 3 sec latency 31.948 ms 2024-09-16T23:58:19.979 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-16T23:58:20.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 453 19.88 MB/sec warmup 4 sec latency 31.891 ms 2024-09-16T23:58:20.884 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:20 smithi007 ceph-mon[23721]: pgmap v242: 97 pgs: 97 active+clean; 20 MiB data, 55 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.9 MiB/s wr, 39 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:20 smithi194 ceph-mon[25713]: pgmap v242: 97 pgs: 97 active+clean; 20 MiB data, 55 MiB used, 715 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.9 MiB/s wr, 39 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:21.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 897 21.69 MB/sec warmup 5 sec latency 82.335 ms 2024-09-16T23:58:22.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 1860 21.96 MB/sec warmup 6 sec latency 67.384 ms 2024-09-16T23:58:22.942 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:22 smithi007 ceph-mon[23721]: pgmap v243: 97 pgs: 97 active+clean; 29 MiB data, 76 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2.8 MiB/s wr, 186 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:23.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:22 smithi194 ceph-mon[25713]: pgmap v243: 97 pgs: 97 active+clean; 29 MiB data, 76 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2.8 MiB/s wr, 186 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:23.239 INFO:teuthology.orchestra.run.smithi007.stdout: 5 2946 23.69 MB/sec warmup 7 sec latency 29.475 ms 2024-09-16T23:58:24.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 3715 24.41 MB/sec warmup 8 sec latency 32.290 ms 2024-09-16T23:58:24.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:24 smithi007 ceph-mon[23721]: pgmap v244: 97 pgs: 97 active+clean; 98 MiB data, 150 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 8.7 MiB/s wr, 411 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:25.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:24 smithi194 ceph-mon[25713]: pgmap v244: 97 pgs: 97 active+clean; 98 MiB data, 150 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 8.7 MiB/s wr, 411 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:25.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 4304 22.45 MB/sec warmup 9 sec latency 20.669 ms 2024-09-16T23:58:25.438 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-16T23:58:25.961 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:25 smithi007 ceph-mon[23721]: pgmap v245: 97 pgs: 97 active+clean; 115 MiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 11 MiB/s wr, 552 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:26.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 5206 22.65 MB/sec warmup 10 sec latency 18.057 ms 2024-09-16T23:58:26.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:25 smithi194 ceph-mon[25713]: pgmap v245: 97 pgs: 97 active+clean; 115 MiB data, 180 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 11 MiB/s wr, 552 op/s; 0 B/s, 0 objects/s recovering 2024-09-16T23:58:27.241 INFO:teuthology.orchestra.run.smithi007.stdout: 5 6038 21.61 MB/sec warmup 11 sec latency 21.343 ms 2024-09-16T23:58:28.224 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:28 smithi007 ceph-mon[23721]: pgmap v246: 97 pgs: 97 active+clean; 131 MiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 13 MiB/s wr, 751 op/s 2024-09-16T23:58:28.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 7011 23.24 MB/sec warmup 12 sec latency 17.582 ms 2024-09-16T23:58:28.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:28 smithi194 ceph-mon[25713]: pgmap v246: 97 pgs: 97 active+clean; 131 MiB data, 194 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 13 MiB/s wr, 751 op/s 2024-09-16T23:58:29.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 7561 22.77 MB/sec warmup 13 sec latency 20.238 ms 2024-09-16T23:58:30.234 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:30 smithi007 ceph-mon[23721]: pgmap v247: 97 pgs: 97 active+clean; 132 MiB data, 263 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 15 MiB/s wr, 839 op/s 2024-09-16T23:58:30.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 8227 21.83 MB/sec warmup 14 sec latency 14.368 ms 2024-09-16T23:58:30.365 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph mgr module enable nfs --force' 2024-09-16T23:58:30.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:30 smithi194 ceph-mon[25713]: pgmap v247: 97 pgs: 97 active+clean; 132 MiB data, 263 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 15 MiB/s wr, 839 op/s 2024-09-16T23:58:31.240 INFO:teuthology.orchestra.run.smithi007.stdout: 5 8982 21.55 MB/sec warmup 15 sec latency 27.142 ms 2024-09-16T23:58:32.216 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:32 smithi007 ceph-mon[23721]: pgmap v248: 97 pgs: 97 active+clean; 140 MiB data, 306 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 14 MiB/s wr, 956 op/s 2024-09-16T23:58:32.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 9575 20.82 MB/sec warmup 16 sec latency 21.693 ms 2024-09-16T23:58:32.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:32 smithi194 ceph-mon[25713]: pgmap v248: 97 pgs: 97 active+clean; 140 MiB data, 306 MiB used, 715 GiB / 715 GiB avail; 7.8 KiB/s rd, 14 MiB/s wr, 956 op/s 2024-09-16T23:58:33.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 9988 20.84 MB/sec warmup 17 sec latency 40.348 ms 2024-09-16T23:58:34.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 10486 20.74 MB/sec warmup 18 sec latency 39.563 ms 2024-09-16T23:58:34.284 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:34 smithi007 ceph-mon[23721]: pgmap v249: 97 pgs: 97 active+clean; 138 MiB data, 372 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 16 MiB/s wr, 1.03k op/s 2024-09-16T23:58:34.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:34 smithi194 ceph-mon[25713]: pgmap v249: 97 pgs: 97 active+clean; 138 MiB data, 372 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 16 MiB/s wr, 1.03k op/s 2024-09-16T23:58:35.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 10862 20.38 MB/sec warmup 19 sec latency 37.693 ms 2024-09-16T23:58:35.955 INFO:teuthology.orchestra.run.smithi007.stderr:module 'nfs' is already enabled 2024-09-16T23:58:36.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:35 smithi007 ceph-mon[23721]: pgmap v250: 97 pgs: 97 active+clean; 145 MiB data, 389 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 956 op/s 2024-09-16T23:58:36.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:35 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3478113799' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-16T23:58:36.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 11220 19.61 MB/sec warmup 20 sec latency 35.579 ms 2024-09-16T23:58:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:35 smithi194 ceph-mon[25713]: pgmap v250: 97 pgs: 97 active+clean; 145 MiB data, 389 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 11 MiB/s wr, 956 op/s 2024-09-16T23:58:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:35 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3478113799' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-16T23:58:37.081 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:36 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3478113799' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-16T23:58:37.081 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:36 smithi007 ceph-mon[23721]: mgrmap e21: smithi007.czdphd(active, since 6m), standbys: smithi194.xlteft 2024-09-16T23:58:37.092 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-16T23:58:37.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 11641 18.94 MB/sec warmup 21 sec latency 39.326 ms 2024-09-16T23:58:37.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:36 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3478113799' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-16T23:58:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:36 smithi194 ceph-mon[25713]: mgrmap e21: smithi007.czdphd(active, since 6m), standbys: smithi194.xlteft 2024-09-16T23:58:38.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:37 smithi007 ceph-mon[23721]: pgmap v251: 97 pgs: 97 active+clean; 148 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 902 op/s 2024-09-16T23:58:38.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 12121 18.83 MB/sec warmup 22 sec latency 50.511 ms 2024-09-16T23:58:38.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:37 smithi194 ceph-mon[25713]: pgmap v251: 97 pgs: 97 active+clean; 148 MiB data, 410 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 10 MiB/s wr, 902 op/s 2024-09-16T23:58:39.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 12700 18.29 MB/sec warmup 23 sec latency 47.778 ms 2024-09-16T23:58:40.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 13040 17.79 MB/sec warmup 24 sec latency 50.131 ms 2024-09-16T23:58:40.383 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:40 smithi007 ceph-mon[23721]: pgmap v252: 97 pgs: 97 active+clean; 155 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 818 op/s 2024-09-16T23:58:40.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:40 smithi194 ceph-mon[25713]: pgmap v252: 97 pgs: 97 active+clean; 155 MiB data, 444 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.2 MiB/s wr, 818 op/s 2024-09-16T23:58:41.004 INFO:teuthology.orchestra.run.smithi007.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-16T23:58:41.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 13295 17.51 MB/sec warmup 25 sec latency 48.788 ms 2024-09-16T23:58:42.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: pgmap v253: 97 pgs: 97 active+clean; 154 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 7.5 MiB/s wr, 735 op/s 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:42 smithi194 ceph-mon[25713]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='client.14590 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: pgmap v253: 97 pgs: 97 active+clean; 154 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 16 KiB/s rd, 7.5 MiB/s wr, 735 op/s 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:58:42.193 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:42 smithi007 ceph-mon[23721]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f 2024-09-16T23:58:42.242 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-16T23:58:42.243 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-16T23:58:42.243 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-16T23:58:42.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 13667 17.45 MB/sec warmup 26 sec latency 39.847 ms 2024-09-16T23:58:43.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 13986 17.29 MB/sec warmup 27 sec latency 60.392 ms 2024-09-16T23:58:44.229 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:44 smithi007 ceph-mon[23721]: pgmap v254: 97 pgs: 97 active+clean; 154 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 739 op/s 2024-09-16T23:58:44.242 INFO:teuthology.orchestra.run.smithi007.stdout: 5 14386 17.23 MB/sec warmup 28 sec latency 41.844 ms 2024-09-16T23:58:44.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:44 smithi194 ceph-mon[25713]: pgmap v254: 97 pgs: 97 active+clean; 154 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.4 MiB/s wr, 739 op/s 2024-09-16T23:58:45.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 14729 16.81 MB/sec warmup 29 sec latency 31.934 ms 2024-09-16T23:58:45.792 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-16T23:58:45.987 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:45 smithi007 ceph-mon[23721]: pgmap v255: 97 pgs: 97 active+clean; 159 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-09-16T23:58:46.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 15370 16.65 MB/sec warmup 30 sec latency 75.728 ms 2024-09-16T23:58:46.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:45 smithi194 ceph-mon[25713]: pgmap v255: 97 pgs: 97 active+clean; 159 MiB data, 464 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 6.3 MiB/s wr, 634 op/s 2024-09-16T23:58:46.903 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (5m) 33s ago 7m 23.2M - ba2b418f427c ddb6debdf7d2 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (6m) 33s ago 6m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (6m) 36s ago 6m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (5m) 33s ago 6m 52.4M - 8.3.5 dad864ee21e9 de7bd16dee04 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (66s) 33s ago 65s 12.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (63s) 36s ago 63s 12.1M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (8m) 33s ago 8m 460M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (6m) 36s ago 6m 414M - 17.2.0 e1d6a67b021e 1223cf6c34e2 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (8m) 33s ago 8m 48.0M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (6m) 36s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (45s) 33s ago 44s 49.2M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-16T23:58:46.904 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (39s) 36s ago 38s 48.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (6m) 33s ago 6m 17.9M - 1dbe0e931976 59ef89f4df45 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 36s ago 6m 17.0M - 1dbe0e931976 1631f9c583ae 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (5m) 33s ago 5m 44.8M 3217M 17.2.0 e1d6a67b021e 78961710495e 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (5m) 33s ago 5m 42.7M 3217M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (4m) 33s ago 4m 39.5M 3217M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (4m) 33s ago 4m 40.1M 3217M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (3m) 36s ago 3m 39.3M 3984M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (3m) 36s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (2m) 36s ago 2m 42.2M 3984M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (2m) 36s ago 2m 40.1M 3984M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-16T23:58:46.905 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (6m) 33s ago 5m 62.8M - 514e6a882f6e c6f8e3619d7a 2024-09-16T23:58:46.992 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:46 smithi007 ceph-mon[23721]: from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:46.992 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:46 smithi007 ceph-mon[23721]: from='client.24399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:47.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 16105 16.60 MB/sec warmup 31 sec latency 16.542 ms 2024-09-16T23:58:47.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:46 smithi194 ceph-mon[25713]: from='client.24395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:47.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:46 smithi194 ceph-mon[25713]: from='client.24399 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:47.575 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:58:47.575 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-16T23:58:47.575 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T23:58:47.576 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-16T23:58:47.577 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:58:47.999 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:47 smithi007 ceph-mon[23721]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:48.000 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:47 smithi007 ceph-mon[23721]: pgmap v256: 97 pgs: 97 active+clean; 167 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.5 MiB/s wr, 568 op/s 2024-09-16T23:58:48.000 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:47 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/4250641795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f", 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "", 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:c2ddf9721ef254645201cdbabd9c8f49c30a586f image" 2024-09-16T23:58:48.146 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:58:48.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 16735 16.49 MB/sec warmup 32 sec latency 24.674 ms 2024-09-16T23:58:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:47 smithi194 ceph-mon[25713]: from='client.14602 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:47 smithi194 ceph-mon[25713]: pgmap v256: 97 pgs: 97 active+clean; 167 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.5 MiB/s wr, 568 op/s 2024-09-16T23:58:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:47 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/4250641795' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T23:58:48.843 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-16T23:58:49.005 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:48 smithi007 ceph-mon[23721]: from='client.24411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:49.005 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:48 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3336858710' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T23:58:49.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 17512 16.97 MB/sec warmup 33 sec latency 14.776 ms 2024-09-16T23:58:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:48 smithi194 ceph-mon[25713]: from='client.24411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:58:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:48 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3336858710' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T23:58:50.010 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:49 smithi007 ceph-mon[23721]: pgmap v257: 97 pgs: 97 active+clean; 171 MiB data, 573 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.0 MiB/s wr, 707 op/s 2024-09-16T23:58:50.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 18060 17.11 MB/sec warmup 34 sec latency 20.432 ms 2024-09-16T23:58:50.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:49 smithi194 ceph-mon[25713]: pgmap v257: 97 pgs: 97 active+clean; 171 MiB data, 573 MiB used, 715 GiB / 715 GiB avail; 6.8 KiB/s rd, 7.0 MiB/s wr, 707 op/s 2024-09-16T23:58:51.243 INFO:teuthology.orchestra.run.smithi007.stdout: 5 18591 16.76 MB/sec warmup 35 sec latency 24.203 ms 2024-09-16T23:58:52.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 19244 16.74 MB/sec warmup 36 sec latency 18.760 ms 2024-09-16T23:58:52.269 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:52 smithi007 ceph-mon[23721]: pgmap v258: 97 pgs: 97 active+clean; 171 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.8 MiB/s wr, 603 op/s 2024-09-16T23:58:52.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:52 smithi194 ceph-mon[25713]: pgmap v258: 97 pgs: 97 active+clean; 171 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 5.8 MiB/s wr, 603 op/s 2024-09-16T23:58:53.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 19904 16.58 MB/sec warmup 37 sec latency 22.977 ms 2024-09-16T23:58:54.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 20751 16.90 MB/sec warmup 38 sec latency 16.755 ms 2024-09-16T23:58:54.282 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:54 smithi007 ceph-mon[23721]: pgmap v259: 97 pgs: 97 active+clean; 179 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.5 MiB/s wr, 841 op/s 2024-09-16T23:58:54.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:54 smithi194 ceph-mon[25713]: pgmap v259: 97 pgs: 97 active+clean; 179 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 8.5 MiB/s wr, 841 op/s 2024-09-16T23:58:55.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 21548 17.31 MB/sec warmup 39 sec latency 51.086 ms 2024-09-16T23:58:56.134 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:55 smithi007 ceph-mon[23721]: pgmap v260: 97 pgs: 97 active+clean; 178 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 740 op/s 2024-09-16T23:58:56.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 21993 17.00 MB/sec warmup 40 sec latency 38.326 ms 2024-09-16T23:58:56.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:55 smithi194 ceph-mon[25713]: pgmap v260: 97 pgs: 97 active+clean; 178 MiB data, 646 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 740 op/s 2024-09-16T23:58:57.244 INFO:teuthology.orchestra.run.smithi007.stdout: 5 22648 16.95 MB/sec warmup 41 sec latency 35.703 ms 2024-09-16T23:58:58.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 23469 16.90 MB/sec warmup 42 sec latency 20.362 ms 2024-09-16T23:58:58.396 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:58:58 smithi007 ceph-mon[23721]: pgmap v261: 97 pgs: 97 active+clean; 180 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 696 op/s 2024-09-16T23:58:58.453 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:58:58 smithi194 ceph-mon[25713]: pgmap v261: 97 pgs: 97 active+clean; 180 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 696 op/s 2024-09-16T23:58:59.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 24057 16.91 MB/sec warmup 43 sec latency 24.484 ms 2024-09-16T23:59:00.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 24718 17.17 MB/sec warmup 44 sec latency 34.152 ms 2024-09-16T23:59:00.414 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:00 smithi007 ceph-mon[23721]: pgmap v262: 97 pgs: 97 active+clean; 191 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 870 op/s 2024-09-16T23:59:00.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:00 smithi194 ceph-mon[25713]: pgmap v262: 97 pgs: 97 active+clean; 191 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.3 MiB/s wr, 870 op/s 2024-09-16T23:59:01.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 25257 17.19 MB/sec warmup 45 sec latency 28.321 ms 2024-09-16T23:59:02.189 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:02 smithi007 ceph-mon[23721]: pgmap v263: 97 pgs: 97 active+clean; 196 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 770 op/s 2024-09-16T23:59:02.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 25702 16.91 MB/sec warmup 46 sec latency 37.491 ms 2024-09-16T23:59:02.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:02 smithi194 ceph-mon[25713]: pgmap v263: 97 pgs: 97 active+clean; 196 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.2 MiB/s wr, 770 op/s 2024-09-16T23:59:03.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 26088 16.73 MB/sec warmup 47 sec latency 77.378 ms 2024-09-16T23:59:04.207 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:04 smithi007 ceph-mon[23721]: pgmap v264: 97 pgs: 97 active+clean; 200 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 928 op/s 2024-09-16T23:59:04.245 INFO:teuthology.orchestra.run.smithi007.stdout: 5 26636 16.69 MB/sec warmup 48 sec latency 27.641 ms 2024-09-16T23:59:04.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:04 smithi194 ceph-mon[25713]: pgmap v264: 97 pgs: 97 active+clean; 200 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 11 MiB/s wr, 928 op/s 2024-09-16T23:59:05.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 27163 16.47 MB/sec warmup 49 sec latency 63.540 ms 2024-09-16T23:59:05.983 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:05 smithi007 ceph-mon[23721]: pgmap v265: 97 pgs: 97 active+clean; 203 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 735 op/s 2024-09-16T23:59:06.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 27854 16.70 MB/sec warmup 50 sec latency 54.978 ms 2024-09-16T23:59:06.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:05 smithi194 ceph-mon[25713]: pgmap v265: 97 pgs: 97 active+clean; 203 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 8.5 MiB/s wr, 735 op/s 2024-09-16T23:59:07.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 28439 16.81 MB/sec warmup 51 sec latency 51.047 ms 2024-09-16T23:59:08.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 28913 16.73 MB/sec warmup 52 sec latency 32.541 ms 2024-09-16T23:59:08.271 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:08 smithi007 ceph-mon[23721]: pgmap v266: 97 pgs: 97 active+clean; 202 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 745 op/s 2024-09-16T23:59:08.452 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:08 smithi194 ceph-mon[25713]: pgmap v266: 97 pgs: 97 active+clean; 202 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.8 MiB/s wr, 745 op/s 2024-09-16T23:59:09.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 29298 16.49 MB/sec warmup 53 sec latency 51.726 ms 2024-09-16T23:59:09.278 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:59:09.279 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:59:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-16T23:59:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-16T23:59:10.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 29549 16.30 MB/sec warmup 54 sec latency 74.087 ms 2024-09-16T23:59:10.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:10 smithi007 ceph-mon[23721]: pgmap v267: 97 pgs: 97 active+clean; 209 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 908 op/s 2024-09-16T23:59:10.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:10 smithi194 ceph-mon[25713]: pgmap v267: 97 pgs: 97 active+clean; 209 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.4 MiB/s wr, 908 op/s 2024-09-16T23:59:11.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 29829 16.12 MB/sec warmup 55 sec latency 101.405 ms 2024-09-16T23:59:12.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 30329 16.04 MB/sec warmup 56 sec latency 52.498 ms 2024-09-16T23:59:12.327 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:12 smithi007 ceph-mon[23721]: pgmap v268: 97 pgs: 97 active+clean; 214 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.6 MiB/s wr, 765 op/s 2024-09-16T23:59:12.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:12 smithi194 ceph-mon[25713]: pgmap v268: 97 pgs: 97 active+clean; 214 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.6 MiB/s wr, 765 op/s 2024-09-16T23:59:13.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 30909 15.97 MB/sec warmup 57 sec latency 50.566 ms 2024-09-16T23:59:14.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 31559 16.18 MB/sec warmup 58 sec latency 50.972 ms 2024-09-16T23:59:14.346 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:14 smithi007 ceph-mon[23721]: pgmap v269: 97 pgs: 97 active+clean; 208 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 700 op/s 2024-09-16T23:59:14.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:14 smithi194 ceph-mon[25713]: pgmap v269: 97 pgs: 97 active+clean; 208 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 8.1 MiB/s wr, 700 op/s 2024-09-16T23:59:15.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 32093 16.26 MB/sec warmup 59 sec latency 33.567 ms 2024-09-16T23:59:16.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 32375 16.08 MB/sec warmup 60 sec latency 50.254 ms 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: pgmap v270: 97 pgs: 97 active+clean; 207 MiB data, 911 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 551 op/s 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (unknown) 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:16.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:16 smithi007 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi007.czdphd) 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: pgmap v270: 97 pgs: 97 active+clean; 207 MiB data, 911 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.0 MiB/s wr, 551 op/s 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: Upgrade: Target is version 19.1.1-330-gc2ddf972 (unknown) 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, digests ['quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3'] 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:16.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:16 smithi194 ceph-mon[25713]: Upgrade: Need to upgrade myself (mgr.smithi007.czdphd) 2024-09-16T23:59:17.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 32785 15.88 MB/sec warmup 61 sec latency 55.060 ms 2024-09-16T23:59:17.655 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:17 smithi007 ceph-mon[23721]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 on smithi194 2024-09-16T23:59:17.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:17 smithi194 ceph-mon[25713]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 on smithi194 2024-09-16T23:59:18.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 33144 15.76 MB/sec warmup 62 sec latency 54.006 ms 2024-09-16T23:59:18.674 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:18 smithi007 ceph-mon[23721]: pgmap v271: 97 pgs: 97 active+clean; 217 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.1 MiB/s wr, 694 op/s 2024-09-16T23:59:18.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:18 smithi194 ceph-mon[25713]: pgmap v271: 97 pgs: 97 active+clean; 217 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.1 MiB/s wr, 694 op/s 2024-09-16T23:59:19.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 33658 15.76 MB/sec warmup 63 sec latency 41.141 ms 2024-09-16T23:59:19.379 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-16T23:59:20.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 34345 15.66 MB/sec warmup 64 sec latency 30.215 ms 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (6m) 67s ago 7m 23.2M - ba2b418f427c ddb6debdf7d2 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (7m) 67s ago 7m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (7m) 69s ago 7m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (6m) 67s ago 7m 52.4M - 8.3.5 dad864ee21e9 de7bd16dee04 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (99s) 67s ago 98s 12.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (96s) 69s ago 96s 12.1M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (8m) 67s ago 8m 460M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (6m) 69s ago 6m 414M - 17.2.0 e1d6a67b021e 1223cf6c34e2 2024-09-16T23:59:20.337 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (8m) 67s ago 8m 48.0M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (6m) 69s ago 6m 37.2M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (78s) 67s ago 77s 49.2M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (72s) 69s ago 72s 48.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (7m) 67s ago 7m 17.9M - 1dbe0e931976 59ef89f4df45 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 69s ago 6m 17.0M - 1dbe0e931976 1631f9c583ae 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (6m) 67s ago 6m 44.8M 3217M 17.2.0 e1d6a67b021e 78961710495e 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (5m) 67s ago 5m 42.7M 3217M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (5m) 67s ago 5m 39.5M 3217M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (4m) 67s ago 4m 40.1M 3217M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (4m) 69s ago 4m 39.3M 3984M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (3m) 69s ago 3m 39.9M 3984M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (3m) 69s ago 3m 42.2M 3984M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-16T23:59:20.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (3m) 69s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-16T23:59:20.339 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (6m) 67s ago 6m 62.8M - 514e6a882f6e c6f8e3619d7a 2024-09-16T23:59:20.667 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:20 smithi007 ceph-mon[23721]: pgmap v272: 97 pgs: 97 active+clean; 229 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.9 MiB/s wr, 825 op/s 2024-09-16T23:59:20.667 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:20 smithi007 ceph-mon[23721]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T23:59:20.798 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-16T23:59:20.799 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:59:20.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:20 smithi194 ceph-mon[25713]: pgmap v272: 97 pgs: 97 active+clean; 229 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 7.9 MiB/s wr, 825 op/s 2024-09-16T23:59:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:20 smithi194 ceph-mon[25713]: from='client.14618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:21.207 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:59:21.208 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-16T23:59:21.208 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-16T23:59:21.208 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-16T23:59:21.208 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "0/23 daemons upgraded", 2024-09-16T23:59:21.208 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 image on host smithi194" 2024-09-16T23:59:21.208 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:59:21.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 34810 15.66 MB/sec warmup 65 sec latency 39.876 ms 2024-09-16T23:59:21.714 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-16T23:59:21.756 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:21 smithi007 ceph-mon[23721]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:21.757 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:21 smithi007 ceph-mon[23721]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:21.757 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:21 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/699468378' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T23:59:21.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:21 smithi194 ceph-mon[25713]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:21.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:21 smithi194 ceph-mon[25713]: from='client.14626 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:21.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:21 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/699468378' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T23:59:22.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 35439 15.84 MB/sec warmup 66 sec latency 29.486 ms 2024-09-16T23:59:22.763 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:22 smithi007 ceph-mon[23721]: pgmap v273: 97 pgs: 97 active+clean; 230 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 646 op/s 2024-09-16T23:59:22.764 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:22 smithi007 ceph-mon[23721]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:22.764 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:22 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3166403040' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T23:59:22.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:22 smithi194 ceph-mon[25713]: pgmap v273: 97 pgs: 97 active+clean; 230 MiB data, 997 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.2 MiB/s wr, 646 op/s 2024-09-16T23:59:22.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:22 smithi194 ceph-mon[25713]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:22.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:22 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3166403040' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T23:59:23.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 35956 15.83 MB/sec warmup 67 sec latency 54.310 ms 2024-09-16T23:59:24.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 36573 15.74 MB/sec warmup 68 sec latency 28.680 ms 2024-09-16T23:59:24.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:24 smithi007 ceph-mon[23721]: pgmap v274: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.0 MiB/s wr, 600 op/s 2024-09-16T23:59:24.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:24 smithi194 ceph-mon[25713]: pgmap v274: 97 pgs: 97 active+clean; 227 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.0 MiB/s wr, 600 op/s 2024-09-16T23:59:25.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 37307 15.78 MB/sec warmup 69 sec latency 24.502 ms 2024-09-16T23:59:26.032 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:25 smithi007 ceph-mon[23721]: pgmap v275: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 649 op/s 2024-09-16T23:59:26.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 38171 15.80 MB/sec warmup 70 sec latency 17.248 ms 2024-09-16T23:59:26.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:25 smithi194 ceph-mon[25713]: pgmap v275: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.7 MiB/s wr, 649 op/s 2024-09-16T23:59:27.247 INFO:teuthology.orchestra.run.smithi007.stdout: 5 39028 16.07 MB/sec warmup 71 sec latency 17.752 ms 2024-09-16T23:59:28.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 39569 16.09 MB/sec warmup 72 sec latency 18.245 ms 2024-09-16T23:59:28.295 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:28 smithi007 ceph-mon[23721]: pgmap v276: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.5 MiB/s wr, 826 op/s 2024-09-16T23:59:28.453 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:28 smithi194 ceph-mon[25713]: pgmap v276: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.5 MiB/s wr, 826 op/s 2024-09-16T23:59:29.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 40257 15.99 MB/sec warmup 73 sec latency 24.493 ms 2024-09-16T23:59:30.213 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:30 smithi007 ceph-mon[23721]: pgmap v277: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 807 op/s 2024-09-16T23:59:30.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 41080 16.07 MB/sec warmup 74 sec latency 16.394 ms 2024-09-16T23:59:30.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:30 smithi194 ceph-mon[25713]: pgmap v277: 97 pgs: 97 active+clean; 244 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 8.7 MiB/s wr, 807 op/s 2024-09-16T23:59:31.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 42005 16.20 MB/sec warmup 75 sec latency 20.661 ms 2024-09-16T23:59:32.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 42719 16.33 MB/sec warmup 76 sec latency 43.170 ms 2024-09-16T23:59:32.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:32 smithi007 ceph-mon[23721]: pgmap v278: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.5 MiB/s wr, 724 op/s 2024-09-16T23:59:32.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:32 smithi194 ceph-mon[25713]: pgmap v278: 97 pgs: 97 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 7.5 MiB/s wr, 724 op/s 2024-09-16T23:59:33.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 42948 16.23 MB/sec warmup 77 sec latency 51.712 ms 2024-09-16T23:59:34.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 43089 16.07 MB/sec warmup 78 sec latency 86.188 ms 2024-09-16T23:59:34.395 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:34 smithi007 ceph-mon[23721]: pgmap v279: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 10 MiB/s wr, 888 op/s 2024-09-16T23:59:34.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:34 smithi194 ceph-mon[25713]: pgmap v279: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.5 KiB/s rd, 10 MiB/s wr, 888 op/s 2024-09-16T23:59:35.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 43315 15.92 MB/sec warmup 79 sec latency 61.393 ms 2024-09-16T23:59:36.007 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:35 smithi007 ceph-mon[23721]: pgmap v280: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 861 op/s 2024-09-16T23:59:36.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 43568 15.74 MB/sec warmup 80 sec latency 110.680 ms 2024-09-16T23:59:36.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:35 smithi194 ceph-mon[25713]: pgmap v280: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 861 op/s 2024-09-16T23:59:37.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 43848 15.60 MB/sec warmup 81 sec latency 72.923 ms 2024-09-16T23:59:38.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 44175 15.54 MB/sec warmup 82 sec latency 48.143 ms 2024-09-16T23:59:38.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:38 smithi007 ceph-mon[23721]: pgmap v281: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 869 op/s 2024-09-16T23:59:38.453 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:38 smithi194 ceph-mon[25713]: pgmap v281: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.5 MiB/s wr, 869 op/s 2024-09-16T23:59:39.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 44703 15.51 MB/sec warmup 83 sec latency 40.751 ms 2024-09-16T23:59:40.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 45176 15.41 MB/sec warmup 84 sec latency 40.451 ms 2024-09-16T23:59:40.317 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:40 smithi007 ceph-mon[23721]: pgmap v282: 97 pgs: 97 active+clean; 258 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 740 op/s 2024-09-16T23:59:40.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:40 smithi194 ceph-mon[25713]: pgmap v282: 97 pgs: 97 active+clean; 258 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 740 op/s 2024-09-16T23:59:41.248 INFO:teuthology.orchestra.run.smithi007.stdout: 5 45470 15.40 MB/sec warmup 85 sec latency 60.959 ms 2024-09-16T23:59:42.253 INFO:teuthology.orchestra.run.smithi007.stdout: 5 45913 15.44 MB/sec warmup 86 sec latency 51.674 ms 2024-09-16T23:59:42.389 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:42 smithi007 ceph-mon[23721]: pgmap v283: 97 pgs: 97 active+clean; 258 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 6.1 MiB/s wr, 569 op/s 2024-09-16T23:59:42.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:42 smithi194 ceph-mon[25713]: pgmap v283: 97 pgs: 97 active+clean; 258 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 6.1 MiB/s wr, 569 op/s 2024-09-16T23:59:43.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 46361 15.41 MB/sec warmup 87 sec latency 42.791 ms 2024-09-16T23:59:44.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 46855 15.37 MB/sec warmup 88 sec latency 38.423 ms 2024-09-16T23:59:44.408 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:44 smithi007 ceph-mon[23721]: pgmap v284: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.5 MiB/s wr, 713 op/s 2024-09-16T23:59:44.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:44 smithi194 ceph-mon[25713]: pgmap v284: 97 pgs: 97 active+clean; 265 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 7.5 MiB/s wr, 713 op/s 2024-09-16T23:59:45.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 47606 15.35 MB/sec warmup 89 sec latency 33.928 ms 2024-09-16T23:59:46.007 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:45 smithi194 ceph-mon[25713]: pgmap v285: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.0 MiB/s wr, 521 op/s 2024-09-16T23:59:46.172 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:45 smithi007 ceph-mon[23721]: pgmap v285: 97 pgs: 97 active+clean; 267 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.0 MiB/s wr, 521 op/s 2024-09-16T23:59:46.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 48572 15.41 MB/sec warmup 90 sec latency 19.794 ms 2024-09-16T23:59:47.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 49292 15.56 MB/sec warmup 91 sec latency 27.752 ms 2024-09-16T23:59:48.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 50092 15.68 MB/sec warmup 92 sec latency 22.172 ms 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: pgmap v286: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: Upgrade: Updating mgr.smithi194.xlteft 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:59:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:48 smithi194 ceph-mon[25713]: Deploying daemon mgr.smithi194.xlteft on smithi194 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: pgmap v286: 97 pgs: 97 active+clean; 268 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.5 MiB/s wr, 516 op/s 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: Upgrade: Updating mgr.smithi194.xlteft 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:59:48.435 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:48 smithi007 ceph-mon[23721]: Deploying daemon mgr.smithi194.xlteft on smithi194 2024-09-16T23:59:49.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 50771 15.61 MB/sec warmup 93 sec latency 22.623 ms 2024-09-16T23:59:50.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 51462 15.63 MB/sec warmup 94 sec latency 54.075 ms 2024-09-16T23:59:50.460 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:50 smithi007 ceph-mon[23721]: pgmap v287: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-09-16T23:59:50.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:50 smithi194 ceph-mon[25713]: pgmap v287: 97 pgs: 97 active+clean; 283 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.1 KiB/s rd, 7.3 MiB/s wr, 736 op/s 2024-09-16T23:59:51.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 52046 15.60 MB/sec warmup 95 sec latency 54.590 ms 2024-09-16T23:59:52.108 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-16T23:59:52.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 52393 15.53 MB/sec warmup 96 sec latency 78.970 ms 2024-09-16T23:59:52.536 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:52 smithi007 ceph-mon[23721]: pgmap v288: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 757 op/s 2024-09-16T23:59:52.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:52 smithi194 ceph-mon[25713]: pgmap v288: 97 pgs: 97 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.3 MiB/s wr, 757 op/s 2024-09-16T23:59:53.066 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-16T23:59:53.066 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (6m) 100s ago 8m 23.2M - ba2b418f427c ddb6debdf7d2 2024-09-16T23:59:53.066 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (8m) 100s ago 8m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-16T23:59:53.066 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (7m) 102s ago 7m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-16T23:59:53.066 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (6m) 100s ago 7m 52.4M - 8.3.5 dad864ee21e9 de7bd16dee04 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (2m) 100s ago 2m 12.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (2m) 102s ago 2m 12.1M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (9m) 100s ago 9m 460M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (7m) 102s ago 7m 414M - 17.2.0 e1d6a67b021e 1223cf6c34e2 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (9m) 100s ago 9m 48.0M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (7m) 102s ago 7m 37.2M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (111s) 100s ago 110s 49.2M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (105s) 102s ago 104s 48.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (7m) 100s ago 7m 17.9M - 1dbe0e931976 59ef89f4df45 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 102s ago 7m 17.0M - 1dbe0e931976 1631f9c583ae 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (6m) 100s ago 6m 44.8M 3217M 17.2.0 e1d6a67b021e 78961710495e 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (6m) 100s ago 6m 42.7M 3217M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (5m) 100s ago 5m 39.5M 3217M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (5m) 100s ago 5m 40.1M 3217M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-16T23:59:53.067 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (4m) 102s ago 4m 39.3M 3984M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-16T23:59:53.068 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (4m) 102s ago 4m 39.9M 3984M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-16T23:59:53.068 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (4m) 102s ago 4m 42.2M 3984M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-16T23:59:53.068 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (3m) 102s ago 3m 40.1M 3984M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-16T23:59:53.068 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (7m) 100s ago 7m 62.8M - 514e6a882f6e c6f8e3619d7a 2024-09-16T23:59:53.260 INFO:teuthology.orchestra.run.smithi007.stdout: 5 52772 15.56 MB/sec warmup 97 sec latency 54.331 ms 2024-09-16T23:59:53.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:53 smithi194 ceph-mon[25713]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:53.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:53 smithi194 ceph-mon[25713]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:53.567 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:53 smithi007 ceph-mon[23721]: from='client.14642 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:53.567 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:53 smithi007 ceph-mon[23721]: from='client.14646 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:53.607 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:59:53.607 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-16T23:59:53.608 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-16T23:59:53.609 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-16T23:59:53.609 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "0/23 daemons upgraded", 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mgr daemons" 2024-09-16T23:59:54.147 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-16T23:59:54.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 53276 15.56 MB/sec warmup 98 sec latency 32.484 ms 2024-09-16T23:59:54.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:54 smithi007 ceph-mon[23721]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:54.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:54 smithi007 ceph-mon[23721]: pgmap v289: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 9.2 MiB/s wr, 887 op/s 2024-09-16T23:59:54.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:54 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3998613939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T23:59:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:54 smithi194 ceph-mon[25713]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:54 smithi194 ceph-mon[25713]: pgmap v289: 97 pgs: 97 active+clean; 282 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 KiB/s rd, 9.2 MiB/s wr, 887 op/s 2024-09-16T23:59:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:54 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3998613939' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-16T23:59:54.696 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-16T23:59:55.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 53642 15.53 MB/sec warmup 99 sec latency 42.270 ms 2024-09-16T23:59:55.362 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:55 smithi007 ceph-mon[23721]: from='client.24445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:55.363 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:55 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3030571918' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T23:59:55.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:55 smithi194 ceph-mon[25713]: from='client.24445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-16T23:59:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:55 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3030571918' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-16T23:59:56.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 54012 15.45 MB/sec warmup 100 sec latency 41.464 ms 2024-09-16T23:59:56.366 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:56 smithi007 ceph-mon[23721]: pgmap v290: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 761 op/s 2024-09-16T23:59:56.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:56 smithi194 ceph-mon[25713]: pgmap v290: 97 pgs: 97 active+clean; 287 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.3 MiB/s wr, 761 op/s 2024-09-16T23:59:57.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 54495 15.36 MB/sec warmup 101 sec latency 32.358 ms 2024-09-16T23:59:58.258 INFO:teuthology.orchestra.run.smithi007.stdout: 5 54946 15.34 MB/sec warmup 102 sec latency 43.151 ms 2024-09-16T23:59:58.375 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:58 smithi007 ceph-mon[23721]: pgmap v291: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 852 op/s 2024-09-16T23:59:58.427 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:58 smithi194 ceph-mon[25713]: pgmap v291: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 852 op/s 2024-09-16T23:59:59.262 INFO:teuthology.orchestra.run.smithi007.stdout: 5 55495 15.30 MB/sec warmup 103 sec latency 44.561 ms 2024-09-16T23:59:59.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:59.666 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:59:59.666 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:59:59.666 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 16 23:59:59 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-16T23:59:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-16T23:59:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-16T23:59:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-16T23:59:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 16 23:59:59 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:00:00.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 55870 15.24 MB/sec warmup 104 sec latency 50.299 ms 2024-09-17T00:00:00.691 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:00 smithi007 ceph-mon[23721]: pgmap v292: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 902 op/s 2024-09-17T00:00:00.691 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:00 smithi007 ceph-mon[23721]: overall HEALTH_OK 2024-09-17T00:00:00.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:00 smithi194 ceph-mon[25713]: pgmap v292: 97 pgs: 97 active+clean; 295 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 9.1 MiB/s wr, 902 op/s 2024-09-17T00:00:00.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:00 smithi194 ceph-mon[25713]: overall HEALTH_OK 2024-09-17T00:00:01.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 56202 15.22 MB/sec warmup 105 sec latency 43.975 ms 2024-09-17T00:00:02.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 56776 15.28 MB/sec warmup 106 sec latency 44.433 ms 2024-09-17T00:00:02.715 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:02 smithi007 ceph-mon[23721]: pgmap v293: 97 pgs: 97 active+clean; 299 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 6.7 MiB/s wr, 655 op/s 2024-09-17T00:00:02.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:02 smithi194 ceph-mon[25713]: pgmap v293: 97 pgs: 97 active+clean; 299 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 6.7 MiB/s wr, 655 op/s 2024-09-17T00:00:03.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 57153 15.26 MB/sec warmup 107 sec latency 38.281 ms 2024-09-17T00:00:04.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 57454 15.18 MB/sec warmup 108 sec latency 60.362 ms 2024-09-17T00:00:04.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: pgmap v294: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.8 MiB/s wr, 727 op/s 2024-09-17T00:00:04.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:04.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:04.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: Standby manager daemon smithi194.xlteft restarted 2024-09-17T00:00:04.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:00:04.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:00:04.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:00:04.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:00:04.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:04 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:00:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: pgmap v294: 97 pgs: 97 active+clean; 303 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.8 MiB/s wr, 727 op/s 2024-09-17T00:00:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft restarted 2024-09-17T00:00:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:00:04.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:00:04.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:00:04.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:00:04.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:04 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:00:05.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 57930 15.07 MB/sec warmup 109 sec latency 110.837 ms 2024-09-17T00:00:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:06 smithi007 ceph-mon[23721]: pgmap v295: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 690 op/s 2024-09-17T00:00:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:06 smithi007 ceph-mon[23721]: mgrmap e22: smithi007.czdphd(active, since 7m), standbys: smithi194.xlteft 2024-09-17T00:00:06.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 58215 15.02 MB/sec warmup 110 sec latency 84.120 ms 2024-09-17T00:00:06.369 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:06 smithi194 ceph-mon[25713]: pgmap v295: 97 pgs: 97 active+clean; 309 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.8 MiB/s wr, 690 op/s 2024-09-17T00:00:06.369 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:06 smithi194 ceph-mon[25713]: mgrmap e22: smithi007.czdphd(active, since 7m), standbys: smithi194.xlteft 2024-09-17T00:00:07.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 58653 15.00 MB/sec warmup 111 sec latency 42.379 ms 2024-09-17T00:00:08.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 59075 14.94 MB/sec warmup 112 sec latency 54.349 ms 2024-09-17T00:00:08.308 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: pgmap v296: 97 pgs: 97 active+clean; 312 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 607 op/s 2024-09-17T00:00:08.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:08.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:08.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:08.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:00:08.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr fail", "who": "smithi007.czdphd"}]: dispatch 2024-09-17T00:00:08.309 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:08 smithi007 ceph-mon[23721]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: pgmap v296: 97 pgs: 97 active+clean; 312 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 7.5 KiB/s rd, 6.7 MiB/s wr, 607 op/s 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr fail", "who": "smithi007.czdphd"}]: dispatch 2024-09-17T00:00:08.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:08 smithi194 ceph-mon[25713]: osdmap e70: 8 total, 8 up, 8 in 2024-09-17T00:00:09.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 59418 14.87 MB/sec warmup 113 sec latency 50.126 ms 2024-09-17T00:00:09.322 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi007.czdphd) 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: Failing over to other MGR 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "mgr fail", "who": "smithi007.czdphd"}]': finished 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: mgrmap e23: smithi194.xlteft(active, starting, since 0.998479s) 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:00:09.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: Manager daemon smithi194.xlteft is now available 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:00:09.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:00:09.325 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/trash_purge_schedule"}]: dispatch 2024-09-17T00:00:09.325 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:09 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/trash_purge_schedule"}]: dispatch 2024-09-17T00:00:09.600 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: Upgrade: Need to upgrade myself (mgr.smithi007.czdphd) 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: Failing over to other MGR 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14200 172.21.15.7:0/2683182456' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "mgr fail", "who": "smithi007.czdphd"}]': finished 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: mgrmap e23: smithi194.xlteft(active, starting, since 0.998479s) 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:00:09.601 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: Manager daemon smithi194.xlteft is now available 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:09.602 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:00:09.603 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:09.603 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:00:09.603 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:00:09.603 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/trash_purge_schedule"}]: dispatch 2024-09-17T00:00:09.603 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:09 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi194.xlteft/trash_purge_schedule"}]: dispatch 2024-09-17T00:00:10.269 INFO:teuthology.orchestra.run.smithi007.stdout: 5 59794 14.86 MB/sec warmup 114 sec latency 52.482 ms 2024-09-17T00:00:10.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:10 smithi007 ceph-mon[23721]: Checking for cert/key for grafana.smithi007 2024-09-17T00:00:10.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:10 smithi007 ceph-mon[23721]: mgrmap e24: smithi194.xlteft(active, since 2s) 2024-09-17T00:00:10.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:10 smithi194 ceph-mon[25713]: Checking for cert/key for grafana.smithi007 2024-09-17T00:00:10.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:10 smithi194 ceph-mon[25713]: mgrmap e24: smithi194.xlteft(active, since 2s) 2024-09-17T00:00:11.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 60159 14.85 MB/sec warmup 115 sec latency 113.904 ms 2024-09-17T00:00:11.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:11 smithi007 ceph-mon[23721]: Deploying cephadm binary to smithi007 2024-09-17T00:00:11.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:11 smithi007 ceph-mon[23721]: Deploying cephadm binary to smithi194 2024-09-17T00:00:11.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:11 smithi007 ceph-mon[23721]: pgmap v3: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:11.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:11 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:11.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:11 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:11.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:11 smithi194 ceph-mon[25713]: Deploying cephadm binary to smithi007 2024-09-17T00:00:11.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:11 smithi194 ceph-mon[25713]: Deploying cephadm binary to smithi194 2024-09-17T00:00:11.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:11 smithi194 ceph-mon[25713]: pgmap v3: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:11.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:11 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:11.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:11 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:12.139 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:00:11 smithi007 sudo[55966]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:12.139 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:00:11 smithi007 sudo[55966]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:12.139 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:00:11 smithi007 sudo[55966]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:12.265 INFO:teuthology.orchestra.run.smithi007.stdout: 5 60492 14.83 MB/sec warmup 116 sec latency 64.049 ms 2024-09-17T00:00:12.384 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:12 smithi007 ceph-mon[23721]: pgmap v4: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:12.384 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:12 smithi007 ceph-mon[23721]: mgrmap e25: smithi194.xlteft(active, since 3s) 2024-09-17T00:00:12.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:12 smithi194 ceph-mon[25713]: pgmap v4: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:12.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:12 smithi194 ceph-mon[25713]: mgrmap e25: smithi194.xlteft(active, since 3s) 2024-09-17T00:00:12.636 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:00:12 smithi007 sudo[56052]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:12.636 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:00:12 smithi007 sudo[56052]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:12.636 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:00:12 smithi007 sudo[56052]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:13.138 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:00:13 smithi007 sudo[56074]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:13.225 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:00:13 smithi007 sudo[56074]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:13.225 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:00:13 smithi007 sudo[56074]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:13.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 60820 14.79 MB/sec warmup 117 sec latency 45.263 ms 2024-09-17T00:00:13.389 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:13 smithi007 ceph-mon[23721]: [17/Sep/2024:00:00:11] ENGINE Bus STARTING 2024-09-17T00:00:13.390 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:13 smithi007 ceph-mon[23721]: [17/Sep/2024:00:00:12] ENGINE Serving on http://172.21.15.194:8765 2024-09-17T00:00:13.390 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:13 smithi007 ceph-mon[23721]: [17/Sep/2024:00:00:12] ENGINE Serving on https://172.21.15.194:7150 2024-09-17T00:00:13.390 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:13 smithi007 ceph-mon[23721]: [17/Sep/2024:00:00:12] ENGINE Bus STARTED 2024-09-17T00:00:13.390 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:13 smithi007 ceph-mon[23721]: [17/Sep/2024:00:00:12] ENGINE Error in HTTPServer.serve 2024-09-17T00:00:13.390 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Traceback (most recent call last): 2024-09-17T00:00:13.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T00:00:13.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._connections.run(self.expiration_interval) 2024-09-17T00:00:13.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T00:00:13.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._run(expiration_interval) 2024-09-17T00:00:13.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T00:00:13.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T00:00:13.392 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T00:00:13.392 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T00:00:13.392 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T00:00:13.392 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: s = self.context.wrap_socket( 2024-09-17T00:00:13.392 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T00:00:13.392 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: return self.sslsocket_class._create( 2024-09-17T00:00:13.393 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T00:00:13.393 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self.do_handshake() 2024-09-17T00:00:13.393 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T00:00:13.393 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._sslobj.do_handshake() 2024-09-17T00:00:13.393 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T00:00:13.393 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:13 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:13 smithi194 ceph-mon[25713]: [17/Sep/2024:00:00:11] ENGINE Bus STARTING 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:13 smithi194 ceph-mon[25713]: [17/Sep/2024:00:00:12] ENGINE Serving on http://172.21.15.194:8765 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:13 smithi194 ceph-mon[25713]: [17/Sep/2024:00:00:12] ENGINE Serving on https://172.21.15.194:7150 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:13 smithi194 ceph-mon[25713]: [17/Sep/2024:00:00:12] ENGINE Bus STARTED 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:13 smithi194 ceph-mon[25713]: [17/Sep/2024:00:00:12] ENGINE Error in HTTPServer.serve 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Traceback (most recent call last): 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._connections.run(self.expiration_interval) 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._run(expiration_interval) 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T00:00:13.462 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s = self.context.wrap_socket( 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: return self.sslsocket_class._create( 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self.do_handshake() 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._sslobj.do_handshake() 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T00:00:13.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:13 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:13.891 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:00:13 smithi007 sudo[56154]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:13.891 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:00:13 smithi007 sudo[56154]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:13.892 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:00:13 smithi007 sudo[56154]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:14.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 61176 14.71 MB/sec warmup 118 sec latency 30.623 ms 2024-09-17T00:00:14.398 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:14 smithi007 ceph-mon[23721]: pgmap v5: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:14.398 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:14 smithi007 ceph-mon[23721]: mgrmap e26: smithi194.xlteft(active, since 5s) 2024-09-17T00:00:14.398 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:00:14 smithi007 sudo[56163]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:14.398 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:00:14 smithi007 sudo[56163]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:14.398 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:00:14 smithi007 sudo[56163]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:14.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:14 smithi194 ceph-mon[25713]: pgmap v5: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:14.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:14 smithi194 ceph-mon[25713]: mgrmap e26: smithi194.xlteft(active, since 5s) 2024-09-17T00:00:14.907 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:00:14 smithi007 sudo[56222]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:14.908 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:00:14 smithi007 sudo[56222]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:14.908 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:00:14 smithi007 sudo[56222]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:15.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 61493 14.62 MB/sec warmup 119 sec latency 40.950 ms 2024-09-17T00:00:15.684 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:00:15 smithi007 sudo[56252]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:15.684 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:00:15 smithi007 sudo[56252]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:15.684 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:00:15 smithi007 sudo[56252]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:16.189 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:00:16 smithi007 sudo[56319]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:16.190 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:00:16 smithi007 sudo[56319]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:16.190 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:00:16 smithi007 sudo[56319]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:16.448 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:16 smithi007 ceph-mon[23721]: pgmap v6: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:16.448 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:16 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.448 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:16 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.448 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:16 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.448 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:16 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.637 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:00:16 smithi194 sudo[43961]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:16.637 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:00:16 smithi194 sudo[43961]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:16.637 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:00:16 smithi194 sudo[43961]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:16.637 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:16 smithi194 ceph-mon[25713]: pgmap v6: 97 pgs: 97 active+clean; 310 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-17T00:00:16.637 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:16 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.637 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:16 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.637 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:16 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.637 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:16 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:16.908 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:00:16 smithi194 sudo[43970]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:16.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:00:16 smithi194 sudo[43970]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:16.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:00:16 smithi194 sudo[43970]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:17.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 62309 18.93 MB/sec execute 1 sec latency 64.801 ms 2024-09-17T00:00:17.519 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[43986]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:17.519 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[43986]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:17.519 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[43986]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:17.824 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[43991]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:17.824 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[43991]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:17.824 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[43991]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:17.971 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:17 smithi007 ceph-mon[23721]: pgmap v7: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 4.1 MiB/s wr, 372 op/s 2024-09-17T00:00:17.972 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:17 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:17 smithi194 ceph-mon[25713]: pgmap v7: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 4.1 MiB/s wr, 372 op/s 2024-09-17T00:00:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:17 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:18.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[44003]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:18.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[44003]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:18.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:00:17 smithi194 sudo[44003]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:18.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 62854 14.27 MB/sec execute 2 sec latency 47.986 ms 2024-09-17T00:00:18.542 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44017]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:18.542 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44017]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:18.543 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44017]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:18.867 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44020]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:18.867 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44020]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:18.867 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44020]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:19.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44023]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:19.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44023]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:19.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:00:18 smithi194 sudo[44023]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:19.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 63172 12.87 MB/sec execute 3 sec latency 336.496 ms 2024-09-17T00:00:19.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 sudo[56557]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:19.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 sudo[56557]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:19.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 sudo[56557]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 sudo[56560]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 sudo[56560]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 sudo[56560]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 ceph-mon[23721]: Standby manager daemon smithi007.czdphd started 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/crt"}]: dispatch 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:00:20.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/key"}]: dispatch 2024-09-17T00:00:20.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:00:20.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:19 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:00:20.076 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:19 smithi194 ceph-mon[25713]: Standby manager daemon smithi007.czdphd started 2024-09-17T00:00:20.076 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:19 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/crt"}]: dispatch 2024-09-17T00:00:20.076 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:19 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:00:20.076 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:19 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/key"}]: dispatch 2024-09-17T00:00:20.076 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:19 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/46750611' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:00:20.076 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:19 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:00:20.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 63405 12.06 MB/sec execute 4 sec latency 862.607 ms 2024-09-17T00:00:20.352 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 sudo[44032]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-17T00:00:20.352 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 sudo[44032]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:20.352 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 sudo[44032]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:20.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 sudo[44035]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-17T00:00:20.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 sudo[44035]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-17T00:00:20.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 sudo[44035]: pam_unix(sudo:session): session closed for user root 2024-09-17T00:00:20.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:00:20.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:00:20.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: pgmap v8: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 3.2 MiB/s wr, 289 op/s 2024-09-17T00:00:20.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: mgrmap e27: smithi194.xlteft(active, since 12s), standbys: smithi007.czdphd 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:20.805 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:20 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: pgmap v8: 97 pgs: 97 active+clean; 324 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 3.2 MiB/s wr, 289 op/s 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: mgrmap e27: smithi194.xlteft(active, since 12s), standbys: smithi007.czdphd 2024-09-17T00:00:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:21.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:20 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:21.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 63824 12.62 MB/sec execute 5 sec latency 67.405 ms 2024-09-17T00:00:22.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 64201 12.74 MB/sec execute 6 sec latency 52.627 ms 2024-09-17T00:00:22.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: pgmap v9: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-17T00:00:22.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:22.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:22.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi007 to 1937M 2024-09-17T00:00:22.333 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:22 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: pgmap v9: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:22.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi007 to 1937M 2024-09-17T00:00:22.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:22 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:23.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 64626 12.30 MB/sec execute 7 sec latency 40.352 ms 2024-09-17T00:00:24.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 64948 11.19 MB/sec execute 8 sec latency 60.463 ms 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: pgmap v10: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.369 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.369 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:24 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: pgmap v10: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-17T00:00:24.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:24 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:25.242 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:00:25.266 INFO:teuthology.orchestra.run.smithi007.stdout: 5 65201 10.49 MB/sec execute 9 sec latency 42.015 ms 2024-09-17T00:00:25.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:25 smithi007 ceph-mon[23721]: Adjusting osd_memory_target on smithi194 to 2704M 2024-09-17T00:00:25.404 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:00:25.404 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:00:25.404 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:25 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-17T00:00:25.404 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:25 smithi007 ceph-mon[23721]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-17T00:00:25.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:25 smithi194 ceph-mon[25713]: Adjusting osd_memory_target on smithi194 to 2704M 2024-09-17T00:00:25.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:00:25.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:00:25.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:25 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-17T00:00:25.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:25 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-17T00:00:26.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 65620 11.08 MB/sec execute 10 sec latency 51.276 ms 2024-09-17T00:00:26.355 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:00:26.355 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (7m) 7s ago 8m 25.1M - 0.23.0 ba2b418f427c ddb6debdf7d2 2024-09-17T00:00:26.355 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (8m) 7s ago 8m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:00:26.355 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (8m) 10s ago 8m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (7m) 7s ago 8m 53.3M - 8.3.5 dad864ee21e9 de7bd16dee04 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (2m) 7s ago 2m 34.2M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (2m) 10s ago 2m 12.9M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (9m) 7s ago 9m 141M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (28s) 10s ago 8m 480M - 19.1.1-330-gc2ddf972 ebb0615149de 6d63d1dc2865 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (9m) 7s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (7m) 10s ago 7m 40.1M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (2m) 7s ago 2m 244M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (2m) 10s ago 2m 75.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (8m) 7s ago 8m 19.1M - 1.3.1 1dbe0e931976 59ef89f4df45 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 10s ago 7m 19.7M - 1.3.1 1dbe0e931976 1631f9c583ae 2024-09-17T00:00:26.356 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (7m) 7s ago 7m 273M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (6m) 7s ago 6m 204M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (6m) 7s ago 6m 188M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (5m) 7s ago 5m 207M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (5m) 10s ago 5m 230M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (5m) 10s ago 5m 258M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (4m) 10s ago 4m 248M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (4m) 10s ago 4m 313M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:00:26.357 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (7m) 7s ago 7m 65.1M - 2.33.4 514e6a882f6e c6f8e3619d7a 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: pgmap v11: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:26.410 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.411 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:26 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: pgmap v11: 97 pgs: 97 active+clean; 325 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 42 KiB/s rd, 4.6 MiB/s wr, 446 op/s 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='client.14680 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:26 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:00:26.924 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:00:26.925 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:00:27.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 66116 10.74 MB/sec execute 11 sec latency 90.896 ms 2024-09-17T00:00:27.388 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:27 smithi007 ceph-mon[23721]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:27.388 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:27 smithi007 ceph-mon[23721]: Reconfiguring alertmanager.smithi007 (dependencies changed)... 2024-09-17T00:00:27.388 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:27 smithi007 ceph-mon[23721]: Deploying daemon alertmanager.smithi007 on smithi007 2024-09-17T00:00:27.388 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:27 smithi007 ceph-mon[23721]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:27.389 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:27 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/1210354727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "", 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:00:27.422 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:00:27.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:27 smithi194 ceph-mon[25713]: from='client.14684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:27.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:27 smithi194 ceph-mon[25713]: Reconfiguring alertmanager.smithi007 (dependencies changed)... 2024-09-17T00:00:27.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:27 smithi194 ceph-mon[25713]: Deploying daemon alertmanager.smithi007 on smithi007 2024-09-17T00:00:27.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:27 smithi194 ceph-mon[25713]: from='client.14688 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:27.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:27 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/1210354727' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:00:27.994 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:00:28.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 66587 10.73 MB/sec execute 12 sec latency 51.260 ms 2024-09-17T00:00:28.395 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:28 smithi007 ceph-mon[23721]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 45 KiB/s rd, 7.1 MiB/s wr, 701 op/s 2024-09-17T00:00:28.395 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:28 smithi007 ceph-mon[23721]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:28.395 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:28 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2948609663' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:00:28.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:28 smithi194 ceph-mon[25713]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 45 KiB/s rd, 7.1 MiB/s wr, 701 op/s 2024-09-17T00:00:28.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:28 smithi194 ceph-mon[25713]: from='client.14696 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:28.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:28 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2948609663' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:00:29.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 67351 12.17 MB/sec execute 13 sec latency 22.084 ms 2024-09-17T00:00:30.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 67936 12.84 MB/sec execute 14 sec latency 20.089 ms 2024-09-17T00:00:30.406 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:30 smithi007 ceph-mon[23721]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-09-17T00:00:30.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:30 smithi194 ceph-mon[25713]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-09-17T00:00:31.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 68442 12.26 MB/sec execute 15 sec latency 46.736 ms 2024-09-17T00:00:32.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 69044 12.61 MB/sec execute 16 sec latency 32.195 ms 2024-09-17T00:00:32.422 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:32 smithi007 ceph-mon[23721]: pgmap v14: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 832 op/s 2024-09-17T00:00:32.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:32 smithi194 ceph-mon[25713]: pgmap v14: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 832 op/s 2024-09-17T00:00:33.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 69838 12.81 MB/sec execute 17 sec latency 21.084 ms 2024-09-17T00:00:34.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 70492 13.33 MB/sec execute 18 sec latency 24.790 ms 2024-09-17T00:00:34.431 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:34 smithi007 ceph-mon[23721]: pgmap v15: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 602 op/s 2024-09-17T00:00:34.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:34 smithi194 ceph-mon[25713]: pgmap v15: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 602 op/s 2024-09-17T00:00:35.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 71242 14.05 MB/sec execute 19 sec latency 21.949 ms 2024-09-17T00:00:36.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 71718 13.86 MB/sec execute 20 sec latency 23.629 ms 2024-09-17T00:00:36.455 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:36 smithi007 ceph-mon[23721]: pgmap v16: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 602 op/s 2024-09-17T00:00:36.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:36 smithi194 ceph-mon[25713]: pgmap v16: 97 pgs: 97 active+clean; 348 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.9 MiB/s wr, 602 op/s 2024-09-17T00:00:37.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 72180 13.47 MB/sec execute 21 sec latency 39.940 ms 2024-09-17T00:00:38.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 72790 13.75 MB/sec execute 22 sec latency 36.983 ms 2024-09-17T00:00:38.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:38 smithi007 ceph-mon[23721]: pgmap v17: 97 pgs: 97 active+clean; 350 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 976 op/s 2024-09-17T00:00:38.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:38 smithi194 ceph-mon[25713]: pgmap v17: 97 pgs: 97 active+clean; 350 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.7 MiB/s wr, 976 op/s 2024-09-17T00:00:39.267 INFO:teuthology.orchestra.run.smithi007.stdout: 5 73555 13.77 MB/sec execute 23 sec latency 35.413 ms 2024-09-17T00:00:39.490 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:39 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:39.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:39 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:40.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 74140 14.04 MB/sec execute 24 sec latency 26.937 ms 2024-09-17T00:00:40.501 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:40 smithi007 ceph-mon[23721]: pgmap v18: 97 pgs: 97 active+clean; 350 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.1 MiB/s wr, 721 op/s 2024-09-17T00:00:40.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:40 smithi194 ceph-mon[25713]: pgmap v18: 97 pgs: 97 active+clean; 350 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 7.1 MiB/s wr, 721 op/s 2024-09-17T00:00:41.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 74621 14.10 MB/sec execute 25 sec latency 54.276 ms 2024-09-17T00:00:42.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 74982 14.15 MB/sec execute 26 sec latency 69.311 ms 2024-09-17T00:00:42.534 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:42 smithi007 ceph-mon[23721]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.06k op/s 2024-09-17T00:00:42.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:42 smithi194 ceph-mon[25713]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 10 MiB/s wr, 1.06k op/s 2024-09-17T00:00:43.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 75185 13.81 MB/sec execute 27 sec latency 88.301 ms 2024-09-17T00:00:44.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 75501 13.38 MB/sec execute 28 sec latency 82.347 ms 2024-09-17T00:00:44.326 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:44 smithi007 ceph-mon[23721]: pgmap v20: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 710 op/s 2024-09-17T00:00:44.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:44 smithi194 ceph-mon[25713]: pgmap v20: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 710 op/s 2024-09-17T00:00:45.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 75745 13.05 MB/sec execute 29 sec latency 92.119 ms 2024-09-17T00:00:46.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 75921 12.76 MB/sec execute 30 sec latency 68.388 ms 2024-09-17T00:00:46.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:46 smithi007 ceph-mon[23721]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 710 op/s 2024-09-17T00:00:46.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:46 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:46 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:46 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:46 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.634 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:46 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:00:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:46 smithi194 ceph-mon[25713]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.1 MiB/s wr, 710 op/s 2024-09-17T00:00:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:46 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:46 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:46 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:46 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:46 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:00:47.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 76147 12.62 MB/sec execute 31 sec latency 61.404 ms 2024-09-17T00:00:47.451 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:47 smithi007 ceph-mon[23721]: Reconfiguring grafana.smithi007 (dependencies changed)... 2024-09-17T00:00:47.451 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:47 smithi007 ceph-mon[23721]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T00:00:47.451 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:47 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:00:47.451 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:47 smithi007 ceph-mon[23721]: Reconfiguring daemon grafana.smithi007 on smithi007 2024-09-17T00:00:47.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:47 smithi194 ceph-mon[25713]: Reconfiguring grafana.smithi007 (dependencies changed)... 2024-09-17T00:00:47.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:47 smithi194 ceph-mon[25713]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-17T00:00:47.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:47 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-17T00:00:47.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:47 smithi194 ceph-mon[25713]: Reconfiguring daemon grafana.smithi007 on smithi007 2024-09-17T00:00:48.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 76560 12.59 MB/sec execute 32 sec latency 46.909 ms 2024-09-17T00:00:48.464 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:48 smithi007 ceph-mon[23721]: pgmap v22: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 920 op/s 2024-09-17T00:00:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:48 smithi194 ceph-mon[25713]: pgmap v22: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 14 KiB/s rd, 9.4 MiB/s wr, 920 op/s 2024-09-17T00:00:49.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 77067 12.43 MB/sec execute 33 sec latency 67.652 ms 2024-09-17T00:00:50.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 77462 12.44 MB/sec execute 34 sec latency 32.658 ms 2024-09-17T00:00:50.438 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:50 smithi007 ceph-mon[23721]: pgmap v23: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 547 op/s 2024-09-17T00:00:50.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:50 smithi194 ceph-mon[25713]: pgmap v23: 97 pgs: 97 active+clean; 346 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.6 MiB/s wr, 547 op/s 2024-09-17T00:00:51.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 78016 12.74 MB/sec execute 35 sec latency 32.618 ms 2024-09-17T00:00:52.268 INFO:teuthology.orchestra.run.smithi007.stdout: 5 78411 12.80 MB/sec execute 36 sec latency 54.687 ms 2024-09-17T00:00:52.452 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:52 smithi007 ceph-mon[23721]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 748 op/s 2024-09-17T00:00:52.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:52 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:52.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:52 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:52 smithi194 ceph-mon[25713]: pgmap v24: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.4 MiB/s wr, 748 op/s 2024-09-17T00:00:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:52 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:52 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:53.269 INFO:teuthology.orchestra.run.smithi007.stdout: 5 78793 12.66 MB/sec execute 37 sec latency 56.494 ms 2024-09-17T00:00:53.461 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:53 smithi007 ceph-mon[23721]: Reconfiguring node-exporter.smithi007 (dependencies changed)... 2024-09-17T00:00:53.461 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:53 smithi007 ceph-mon[23721]: Deploying daemon node-exporter.smithi007 on smithi007 2024-09-17T00:00:53.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:53 smithi194 ceph-mon[25713]: Reconfiguring node-exporter.smithi007 (dependencies changed)... 2024-09-17T00:00:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:53 smithi194 ceph-mon[25713]: Deploying daemon node-exporter.smithi007 on smithi007 2024-09-17T00:00:54.269 INFO:teuthology.orchestra.run.smithi007.stdout: 5 79209 12.43 MB/sec execute 38 sec latency 43.842 ms 2024-09-17T00:00:54.495 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:54 smithi007 ceph-mon[23721]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-17T00:00:54.495 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:54 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:54.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:54 smithi194 ceph-mon[25713]: pgmap v25: 97 pgs: 97 active+clean; 327 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.1 MiB/s wr, 411 op/s 2024-09-17T00:00:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:54 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:00:55.269 INFO:teuthology.orchestra.run.smithi007.stdout: 5 79486 12.27 MB/sec execute 39 sec latency 93.985 ms 2024-09-17T00:00:56.269 INFO:teuthology.orchestra.run.smithi007.stdout: 5 79887 12.34 MB/sec execute 40 sec latency 43.173 ms 2024-09-17T00:00:56.538 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:56 smithi007 ceph-mon[23721]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 427 op/s 2024-09-17T00:00:56.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:56 smithi194 ceph-mon[25713]: pgmap v26: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.6 MiB/s wr, 427 op/s 2024-09-17T00:00:57.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 80374 12.22 MB/sec execute 41 sec latency 44.675 ms 2024-09-17T00:00:58.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 80785 12.17 MB/sec execute 42 sec latency 43.407 ms 2024-09-17T00:00:58.571 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:00:58.587 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:58 smithi007 ceph-mon[23721]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.0 MiB/s wr, 664 op/s 2024-09-17T00:00:58.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:58 smithi194 ceph-mon[25713]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 7.0 MiB/s wr, 664 op/s 2024-09-17T00:00:59.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 81149 12.26 MB/sec execute 43 sec latency 32.991 ms 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 starting - - - - 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (9m) 40s ago 9m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (8m) 44s ago 8m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 starting - - - - 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (3m) 40s ago 3m 34.2M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (3m) 44s ago 3m 12.9M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (10m) 40s ago 10m 141M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (61s) 44s ago 8m 480M - 19.1.1-330-gc2ddf972 ebb0615149de 6d63d1dc2865 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (10m) 40s ago 10m 52.3M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (8m) 44s ago 8m 40.1M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-17T00:00:59.619 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (2m) 40s ago 2m 244M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (2m) 44s ago 2m 75.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 starting - - - - 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (8m) 44s ago 8m 19.7M - 1.3.1 1dbe0e931976 1631f9c583ae 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (7m) 40s ago 7m 273M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (7m) 40s ago 7m 204M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (6m) 40s ago 6m 188M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (6m) 40s ago 6m 207M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (5m) 44s ago 5m 230M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (5m) 44s ago 5m 258M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (5m) 44s ago 5m 248M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (4m) 44s ago 4m 313M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:00:59.620 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (8m) 40s ago 8m 65.1M - 2.33.4 514e6a882f6e c6f8e3619d7a 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: Reconfiguring prometheus.smithi007 (dependencies changed)... 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:00:59.852 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:00:59 smithi007 ceph-mon[23721]: Deploying daemon prometheus.smithi007 on smithi007 2024-09-17T00:01:00.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: from='client.24503 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: Reconfiguring prometheus.smithi007 (dependencies changed)... 2024-09-17T00:01:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.7 MiB/s wr, 453 op/s 2024-09-17T00:01:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: from='client.14708 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:00:59 smithi194 ceph-mon[25713]: Deploying daemon prometheus.smithi007 on smithi007 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:00.269 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:01:00.270 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:01:00.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 81585 12.31 MB/sec execute 44 sec latency 35.568 ms 2024-09-17T00:01:00.840 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:01:00.840 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:01:00.840 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:01:00.841 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:01:00.841 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-17T00:01:00.841 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T00:01:00.841 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "", 2024-09-17T00:01:00.841 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:01:00.841 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:01:01.124 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:00 smithi007 ceph-mon[23721]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:01.124 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:00 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2164656938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:01:01.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 82016 12.41 MB/sec execute 45 sec latency 63.885 ms 2024-09-17T00:01:01.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:00 smithi194 ceph-mon[25713]: from='client.14712 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:01.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:00 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2164656938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:01:01.510 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:01:02.136 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:01 smithi007 ceph-mon[23721]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:02.136 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:01 smithi007 ceph-mon[23721]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.2 MiB/s wr, 690 op/s 2024-09-17T00:01:02.136 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:01 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/317358' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:01:02.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 82352 12.26 MB/sec execute 46 sec latency 45.560 ms 2024-09-17T00:01:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:01 smithi194 ceph-mon[25713]: from='client.14720 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:01 smithi194 ceph-mon[25713]: pgmap v29: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 7.2 MiB/s wr, 690 op/s 2024-09-17T00:01:02.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:01 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/317358' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:01:03.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 82813 12.10 MB/sec execute 47 sec latency 42.114 ms 2024-09-17T00:01:04.155 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:03 smithi007 ceph-mon[23721]: pgmap v30: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 489 op/s 2024-09-17T00:01:04.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 83324 12.25 MB/sec execute 48 sec latency 44.523 ms 2024-09-17T00:01:04.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:03 smithi194 ceph-mon[25713]: pgmap v30: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 489 op/s 2024-09-17T00:01:05.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 83936 12.16 MB/sec execute 49 sec latency 54.213 ms 2024-09-17T00:01:06.182 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:05 smithi007 ceph-mon[23721]: pgmap v31: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 508 op/s 2024-09-17T00:01:06.275 INFO:teuthology.orchestra.run.smithi007.stdout: 5 84326 12.12 MB/sec execute 50 sec latency 61.746 ms 2024-09-17T00:01:06.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:05 smithi194 ceph-mon[25713]: pgmap v31: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.8 MiB/s wr, 508 op/s 2024-09-17T00:01:07.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 84477 11.96 MB/sec execute 51 sec latency 84.750 ms 2024-09-17T00:01:08.025 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:07 smithi007 ceph-mon[23721]: pgmap v32: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 747 op/s 2024-09-17T00:01:08.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 84944 12.11 MB/sec execute 52 sec latency 53.925 ms 2024-09-17T00:01:08.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:07 smithi194 ceph-mon[25713]: pgmap v32: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 747 op/s 2024-09-17T00:01:09.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 85279 12.13 MB/sec execute 53 sec latency 102.691 ms 2024-09-17T00:01:10.096 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:09 smithi007 ceph-mon[23721]: pgmap v33: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-09-17T00:01:10.097 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:09 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:10.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 85596 12.12 MB/sec execute 54 sec latency 50.530 ms 2024-09-17T00:01:10.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:09 smithi194 ceph-mon[25713]: pgmap v33: 97 pgs: 97 active+clean; 337 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 510 op/s 2024-09-17T00:01:10.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:09 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:11.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 85821 11.99 MB/sec execute 55 sec latency 52.739 ms 2024-09-17T00:01:12.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:11 smithi007 ceph-mon[23721]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 722 op/s 2024-09-17T00:01:12.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 86243 11.84 MB/sec execute 56 sec latency 127.622 ms 2024-09-17T00:01:12.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:11 smithi194 ceph-mon[25713]: pgmap v34: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 722 op/s 2024-09-17T00:01:13.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 86600 11.77 MB/sec execute 57 sec latency 54.430 ms 2024-09-17T00:01:14.160 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:13 smithi007 ceph-mon[23721]: pgmap v35: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 486 op/s 2024-09-17T00:01:14.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 87097 11.87 MB/sec execute 58 sec latency 37.172 ms 2024-09-17T00:01:14.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:13 smithi194 ceph-mon[25713]: pgmap v35: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.1 MiB/s wr, 486 op/s 2024-09-17T00:01:15.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 87626 11.79 MB/sec execute 59 sec latency 80.010 ms 2024-09-17T00:01:16.200 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:15 smithi007 ceph-mon[23721]: pgmap v36: 97 pgs: 97 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 556 op/s 2024-09-17T00:01:16.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 87898 11.71 MB/sec execute 60 sec latency 64.437 ms 2024-09-17T00:01:16.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:15 smithi194 ceph-mon[25713]: pgmap v36: 97 pgs: 97 active+clean; 339 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 556 op/s 2024-09-17T00:01:17.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 88288 11.82 MB/sec execute 61 sec latency 43.592 ms 2024-09-17T00:01:17.991 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:17 smithi007 ceph-mon[23721]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 703 op/s 2024-09-17T00:01:18.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 88752 11.85 MB/sec execute 62 sec latency 29.488 ms 2024-09-17T00:01:18.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:17 smithi194 ceph-mon[25713]: pgmap v37: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.6 MiB/s wr, 703 op/s 2024-09-17T00:01:19.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 89160 11.92 MB/sec execute 63 sec latency 31.673 ms 2024-09-17T00:01:20.033 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:19 smithi007 ceph-mon[23721]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-17T00:01:20.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 89494 11.81 MB/sec execute 64 sec latency 42.787 ms 2024-09-17T00:01:20.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:19 smithi194 ceph-mon[25713]: pgmap v38: 97 pgs: 97 active+clean; 335 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-17T00:01:21.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 89881 11.68 MB/sec execute 65 sec latency 53.914 ms 2024-09-17T00:01:21.541 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:21 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:21.541 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:21 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:21.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:21 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:21.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:21 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:22.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 90164 11.61 MB/sec execute 66 sec latency 83.124 ms 2024-09-17T00:01:22.571 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:22 smithi007 ceph-mon[23721]: Reconfiguring node-exporter.smithi194 (dependencies changed)... 2024-09-17T00:01:22.571 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:22 smithi007 ceph-mon[23721]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-17T00:01:22.571 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:22 smithi007 ceph-mon[23721]: pgmap v39: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 681 op/s 2024-09-17T00:01:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:22 smithi194 ceph-mon[25713]: Reconfiguring node-exporter.smithi194 (dependencies changed)... 2024-09-17T00:01:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:22 smithi194 ceph-mon[25713]: Deploying daemon node-exporter.smithi194 on smithi194 2024-09-17T00:01:22.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:22 smithi194 ceph-mon[25713]: pgmap v39: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 9.7 KiB/s rd, 7.3 MiB/s wr, 681 op/s 2024-09-17T00:01:23.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 90591 11.68 MB/sec execute 67 sec latency 65.113 ms 2024-09-17T00:01:23.584 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:23 smithi007 ceph-mon[23721]: pgmap v40: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-09-17T00:01:23.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:23 smithi194 ceph-mon[25713]: pgmap v40: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.8 MiB/s wr, 469 op/s 2024-09-17T00:01:24.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 91046 11.56 MB/sec execute 68 sec latency 63.470 ms 2024-09-17T00:01:24.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:24 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:24.755 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:24 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:25.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 91356 11.53 MB/sec execute 69 sec latency 50.015 ms 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 570 op/s 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi007.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi007.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T00:01:26.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.169 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:01:26.169 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi007.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T00:01:26.169 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.169 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:25 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:26.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 91601 11.53 MB/sec execute 70 sec latency 85.337 ms 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: pgmap v41: 97 pgs: 97 active+clean; 340 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.6 MiB/s wr, 570 op/s 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi007.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi007.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T00:01:26.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:01:26.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi007.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T00:01:26.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:26.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:25 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi007.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi007.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi007.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi194.xlteft) 2024-09-17T00:01:27.210 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:26 smithi007 ceph-mon[23721]: Upgrade: Need to upgrade myself (mgr.smithi194.xlteft) 2024-09-17T00:01:27.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 91987 11.56 MB/sec execute 71 sec latency 72.542 ms 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi007.front.sepia.ceph.com:9093"}]: dispatch 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi007.front.sepia.ceph.com:3000"}]: dispatch 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi007.front.sepia.ceph.com:9095"}]: dispatch 2024-09-17T00:01:27.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: Upgrade: Need to upgrade myself (mgr.smithi194.xlteft) 2024-09-17T00:01:27.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:26 smithi194 ceph-mon[25713]: Upgrade: Need to upgrade myself (mgr.smithi194.xlteft) 2024-09-17T00:01:27.975 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:27 smithi007 ceph-mon[23721]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 624 op/s 2024-09-17T00:01:28.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 92341 11.58 MB/sec execute 72 sec latency 70.144 ms 2024-09-17T00:01:28.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:27 smithi194 ceph-mon[25713]: pgmap v42: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.0 MiB/s wr, 624 op/s 2024-09-17T00:01:29.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 92653 11.59 MB/sec execute 73 sec latency 48.019 ms 2024-09-17T00:01:30.029 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:29 smithi007 ceph-mon[23721]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 458 op/s 2024-09-17T00:01:30.276 INFO:teuthology.orchestra.run.smithi007.stdout: 5 92903 11.50 MB/sec execute 74 sec latency 79.884 ms 2024-09-17T00:01:30.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:29 smithi194 ceph-mon[25713]: pgmap v43: 97 pgs: 97 active+clean; 332 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.7 MiB/s wr, 458 op/s 2024-09-17T00:01:31.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 93224 11.37 MB/sec execute 75 sec latency 55.294 ms 2024-09-17T00:01:31.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: Upgrade: Updating mgr.smithi007.czdphd 2024-09-17T00:01:31.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:31.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:01:31.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:01:31.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:01:31.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:01:31.324 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:31 smithi007 ceph-mon[23721]: Deploying daemon mgr.smithi007.czdphd on smithi007 2024-09-17T00:01:31.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: Upgrade: Updating mgr.smithi007.czdphd 2024-09-17T00:01:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:01:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:01:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:01:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:01:31.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:31 smithi194 ceph-mon[25713]: Deploying daemon mgr.smithi007.czdphd on smithi007 2024-09-17T00:01:31.993 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:01:32.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 93491 11.26 MB/sec execute 76 sec latency 49.408 ms 2024-09-17T00:01:32.330 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:32 smithi007 ceph-mon[23721]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-09-17T00:01:32.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:32 smithi194 ceph-mon[25713]: pgmap v44: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-09-17T00:01:33.027 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:01:33.027 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 starting - - - - 2024-09-17T00:01:33.027 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (9m) 74s ago 9m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:01:33.027 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (9m) 77s ago 9m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:01:33.027 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 starting - - - - 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (3m) 74s ago 3m 34.2M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (3m) 77s ago 3m 12.9M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:9283 running (11m) 74s ago 11m 141M - 17.2.0 e1d6a67b021e 2348b6eb3db0 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (94s) 77s ago 9m 480M - 19.1.1-330-gc2ddf972 ebb0615149de 6d63d1dc2865 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (11m) 74s ago 11m 52.3M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (9m) 77s ago 9m 40.1M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (3m) 74s ago 3m 244M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (3m) 77s ago 3m 75.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 starting - - - - 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 starting - - - - 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (8m) 74s ago 8m 273M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (7m) 74s ago 7m 204M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:01:33.028 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (7m) 74s ago 7m 188M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:01:33.029 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (6m) 74s ago 6m 207M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:01:33.029 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (6m) 77s ago 6m 230M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:01:33.029 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (6m) 77s ago 6m 258M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:01:33.029 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (5m) 77s ago 5m 248M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:01:33.029 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (5m) 77s ago 5m 313M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:01:33.029 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 starting - - - - 2024-09-17T00:01:33.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 93740 11.19 MB/sec execute 77 sec latency 68.004 ms 2024-09-17T00:01:33.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:33 smithi007 ceph-mon[23721]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:33.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:33 smithi007 ceph-mon[23721]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:33.620 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:01:33.620 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:01:33.620 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:01:33.620 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:01:33.621 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:01:33.622 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-17T00:01:33.622 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:01:33.622 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:01:33.622 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:01:33.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:33 smithi194 ceph-mon[25713]: from='client.14728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:33.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:33 smithi194 ceph-mon[25713]: from='client.24529 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:34.098 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:01:34.098 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "1/23 daemons upgraded", 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mgr daemons", 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:01:34.099 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:01:34.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 94152 11.26 MB/sec execute 78 sec latency 47.314 ms 2024-09-17T00:01:34.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:34 smithi007 ceph-mon[23721]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 420 op/s 2024-09-17T00:01:34.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:34 smithi007 ceph-mon[23721]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:34.368 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:34 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/3595676477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:01:34.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:34 smithi194 ceph-mon[25713]: pgmap v45: 97 pgs: 97 active+clean; 342 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.4 MiB/s wr, 420 op/s 2024-09-17T00:01:34.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:34 smithi194 ceph-mon[25713]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:34.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:34 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/3595676477' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:01:34.705 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:01:35.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 94577 11.16 MB/sec execute 79 sec latency 55.963 ms 2024-09-17T00:01:35.400 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:35 smithi007 ceph-mon[23721]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:35.400 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:35 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2059677840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:01:35.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:35 smithi194 ceph-mon[25713]: from='client.14744 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:01:35.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:35 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2059677840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:01:36.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 94899 11.14 MB/sec execute 80 sec latency 61.160 ms 2024-09-17T00:01:36.424 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:36 smithi007 ceph-mon[23721]: pgmap v46: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.6 MiB/s wr, 605 op/s 2024-09-17T00:01:36.424 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:36 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:36.424 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:36 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:36.424 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:36 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:36 smithi194 ceph-mon[25713]: pgmap v46: 97 pgs: 97 active+clean; 331 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.6 MiB/s wr, 605 op/s 2024-09-17T00:01:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:36 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:36.661 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:36 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:36.661 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:36 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:37.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 95017 11.05 MB/sec execute 81 sec latency 59.228 ms 2024-09-17T00:01:37.723 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:37 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:37.723 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:37 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:37.723 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:37 smithi007 ceph-mon[23721]: pgmap v47: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.0 MiB/s wr, 516 op/s 2024-09-17T00:01:37.821 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:37 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:37.821 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:37 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:37.821 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:37 smithi194 ceph-mon[25713]: pgmap v47: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.0 MiB/s wr, 516 op/s 2024-09-17T00:01:38.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 95408 11.15 MB/sec execute 82 sec latency 60.854 ms 2024-09-17T00:01:39.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 95796 11.12 MB/sec execute 83 sec latency 53.466 ms 2024-09-17T00:01:40.026 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:39 smithi007 ceph-mon[23721]: pgmap v48: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 391 op/s 2024-09-17T00:01:40.026 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:39 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:40.026 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:39 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:40.026 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:39 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:40.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 96130 11.16 MB/sec execute 84 sec latency 50.894 ms 2024-09-17T00:01:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:39 smithi194 ceph-mon[25713]: pgmap v48: 97 pgs: 97 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 391 op/s 2024-09-17T00:01:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:39 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:39 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:39 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:41.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 96298 11.09 MB/sec execute 85 sec latency 93.076 ms 2024-09-17T00:01:42.069 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:41 smithi007 ceph-mon[23721]: pgmap v49: 97 pgs: 97 active+clean; 347 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-09-17T00:01:42.070 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:41 smithi007 ceph-mon[23721]: mgrmap e28: smithi194.xlteft(active, since 93s), standbys: smithi007.czdphd 2024-09-17T00:01:42.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 96548 11.01 MB/sec execute 86 sec latency 41.533 ms 2024-09-17T00:01:42.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:41 smithi194 ceph-mon[25713]: pgmap v49: 97 pgs: 97 active+clean; 347 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 572 op/s 2024-09-17T00:01:42.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:41 smithi194 ceph-mon[25713]: mgrmap e28: smithi194.xlteft(active, since 93s), standbys: smithi007.czdphd 2024-09-17T00:01:43.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 96850 10.91 MB/sec execute 87 sec latency 63.322 ms 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: pgmap v50: 97 pgs: 97 active+clean; 347 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 377 op/s 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: Standby manager daemon smithi007.czdphd restarted 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: Standby manager daemon smithi007.czdphd started 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/crt"}]: dispatch 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/key"}]: dispatch 2024-09-17T00:01:44.113 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:43 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:01:44.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 97129 10.85 MB/sec execute 88 sec latency 60.056 ms 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: pgmap v50: 97 pgs: 97 active+clean; 347 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 377 op/s 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: Standby manager daemon smithi007.czdphd restarted 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: Standby manager daemon smithi007.czdphd started 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/crt"}]: dispatch 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi007.czdphd/key"}]: dispatch 2024-09-17T00:01:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:43 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:01:45.139 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:45 smithi007 ceph-mon[23721]: mgrmap e29: smithi194.xlteft(active, since 96s), standbys: smithi007.czdphd 2024-09-17T00:01:45.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 97506 10.90 MB/sec execute 89 sec latency 44.666 ms 2024-09-17T00:01:45.414 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:45 smithi194 ceph-mon[25713]: mgrmap e29: smithi194.xlteft(active, since 96s), standbys: smithi007.czdphd 2024-09-17T00:01:46.277 INFO:teuthology.orchestra.run.smithi007.stdout: 5 97953 10.84 MB/sec execute 90 sec latency 54.882 ms 2024-09-17T00:01:46.397 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:46 smithi007 ceph-mon[23721]: pgmap v51: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 533 op/s 2024-09-17T00:01:46.397 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:46 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:46.397 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:46 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:46.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:46 smithi194 ceph-mon[25713]: pgmap v51: 97 pgs: 97 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.3 MiB/s wr, 533 op/s 2024-09-17T00:01:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:46 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:46.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:46 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:47.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 98377 10.81 MB/sec execute 91 sec latency 39.360 ms 2024-09-17T00:01:47.416 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:47 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:47.416 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:47 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:47.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:47 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:47.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:47 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:48.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 98680 10.82 MB/sec execute 92 sec latency 44.022 ms 2024-09-17T00:01:48.437 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:48 smithi007 ceph-mon[23721]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.2 MiB/s wr, 379 op/s 2024-09-17T00:01:48.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:48 smithi194 ceph-mon[25713]: pgmap v52: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.8 KiB/s rd, 4.2 MiB/s wr, 379 op/s 2024-09-17T00:01:49.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 99027 10.88 MB/sec execute 93 sec latency 51.831 ms 2024-09-17T00:01:50.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 99385 10.87 MB/sec execute 94 sec latency 55.368 ms 2024-09-17T00:01:50.472 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:50 smithi007 ceph-mon[23721]: pgmap v53: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 367 op/s 2024-09-17T00:01:50.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:50 smithi194 ceph-mon[25713]: pgmap v53: 97 pgs: 97 active+clean; 334 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 367 op/s 2024-09-17T00:01:51.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 99680 10.88 MB/sec execute 95 sec latency 68.157 ms 2024-09-17T00:01:52.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 99799 10.81 MB/sec execute 96 sec latency 64.220 ms 2024-09-17T00:01:52.521 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: pgmap v54: 97 pgs: 97 active+clean; 346 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.4 MiB/s wr, 577 op/s 2024-09-17T00:01:52.521 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.521 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.521 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:01:52.521 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:01:52.521 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.522 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:52 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: pgmap v54: 97 pgs: 97 active+clean; 346 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.4 MiB/s wr, 577 op/s 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:52 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' 2024-09-17T00:01:53.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 99920 10.71 MB/sec execute 97 sec latency 161.008 ms 2024-09-17T00:01:53.563 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:53 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:53.563 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:53 smithi007 ceph-mon[23721]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr fail", "who": "smithi194.xlteft"}]: dispatch 2024-09-17T00:01:53.563 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:53 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr fail", "who": "smithi194.xlteft"}]: dispatch 2024-09-17T00:01:53.563 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:53 smithi007 ceph-mon[23721]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T00:01:53.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:53 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:53.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:53 smithi194 ceph-mon[25713]: from='mgr.14666 172.21.15.194:0/1825431568' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr fail", "who": "smithi194.xlteft"}]: dispatch 2024-09-17T00:01:53.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:53 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "mgr fail", "who": "smithi194.xlteft"}]: dispatch 2024-09-17T00:01:53.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:53 smithi194 ceph-mon[25713]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T00:01:54.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 100145 10.64 MB/sec execute 98 sec latency 110.716 ms 2024-09-17T00:01:54.639 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd='[{"prefix": "mgr fail", "who": "smithi194.xlteft"}]': finished 2024-09-17T00:01:54.639 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: mgrmap e30: smithi007.czdphd(active, starting, since 0.889637s) 2024-09-17T00:01:54.639 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:01:54.639 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:01:54.640 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:01:54.641 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:01:54.641 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: Manager daemon smithi007.czdphd is now available 2024-09-17T00:01:54.641 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:54.641 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:54.641 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:01:54.641 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:54 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-17T00:01:54.867 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14666 ' entity='mgr.smithi194.xlteft' cmd='[{"prefix": "mgr fail", "who": "smithi194.xlteft"}]': finished 2024-09-17T00:01:54.867 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: mgrmap e30: smithi007.czdphd(active, starting, since 0.889637s) 2024-09-17T00:01:54.867 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:01:54.867 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:01:54.867 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:01:54.867 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: Manager daemon smithi007.czdphd is now available 2024-09-17T00:01:54.868 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:01:54.869 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:01:54.869 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:01:54.869 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:54 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-17T00:01:55.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 100448 10.56 MB/sec execute 99 sec latency 56.909 ms 2024-09-17T00:01:55.645 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:55 smithi007 ceph-mon[23721]: mgrmap e31: smithi007.czdphd(active, since 2s) 2024-09-17T00:01:55.645 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:55 smithi007 ceph-mon[23721]: pgmap v3: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:01:55.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:55 smithi194 ceph-mon[25713]: mgrmap e31: smithi007.czdphd(active, since 2s) 2024-09-17T00:01:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:55 smithi194 ceph-mon[25713]: pgmap v3: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:01:56.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 100701 10.51 MB/sec execute 100 sec latency 51.497 ms 2024-09-17T00:01:56.703 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:56 smithi007 ceph-mon[23721]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:01:56.704 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:56 smithi007 ceph-mon[23721]: mgrmap e32: smithi007.czdphd(active, since 3s) 2024-09-17T00:01:56.880 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:56 smithi194 ceph-mon[25713]: pgmap v4: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:01:56.880 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:56 smithi194 ceph-mon[25713]: mgrmap e32: smithi007.czdphd(active, since 3s) 2024-09-17T00:01:57.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 100976 10.49 MB/sec execute 101 sec latency 75.891 ms 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:57 smithi007 ceph-mon[23721]: [17/Sep/2024:00:01:56] ENGINE Bus STARTING 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:57 smithi007 ceph-mon[23721]: [17/Sep/2024:00:01:56] ENGINE Serving on https://172.21.15.7:7150 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:57 smithi007 ceph-mon[23721]: [17/Sep/2024:00:01:56] ENGINE Error in HTTPServer.serve 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Traceback (most recent call last): 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._connections.run(self.expiration_interval) 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._run(expiration_interval) 2024-09-17T00:01:57.726 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: s = self.context.wrap_socket( 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: return self.sslsocket_class._create( 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self.do_handshake() 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._sslobj.do_handshake() 2024-09-17T00:01:57.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T00:01:57.798 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:01:56] ENGINE Bus STARTING 2024-09-17T00:01:57.798 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:01:56] ENGINE Serving on https://172.21.15.7:7150 2024-09-17T00:01:57.798 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:01:56] ENGINE Error in HTTPServer.serve 2024-09-17T00:01:57.798 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Traceback (most recent call last): 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._connections.run(self.expiration_interval) 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._run(expiration_interval) 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s = self.context.wrap_socket( 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T00:01:57.799 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: return self.sslsocket_class._create( 2024-09-17T00:01:57.800 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T00:01:57.800 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self.do_handshake() 2024-09-17T00:01:57.800 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T00:01:57.800 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._sslobj.do_handshake() 2024-09-17T00:01:57.800 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T00:01:58.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 101392 10.50 MB/sec execute 102 sec latency 53.287 ms 2024-09-17T00:01:58.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:58 smithi007 ceph-mon[23721]: [17/Sep/2024:00:01:56] ENGINE Serving on http://172.21.15.7:8765 2024-09-17T00:01:58.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:58 smithi007 ceph-mon[23721]: [17/Sep/2024:00:01:56] ENGINE Bus STARTED 2024-09-17T00:01:58.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:58 smithi007 ceph-mon[23721]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:01:58.744 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:58 smithi007 ceph-mon[23721]: mgrmap e33: smithi007.czdphd(active, since 5s) 2024-09-17T00:01:58.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:58 smithi194 ceph-mon[25713]: [17/Sep/2024:00:01:56] ENGINE Serving on http://172.21.15.7:8765 2024-09-17T00:01:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:58 smithi194 ceph-mon[25713]: [17/Sep/2024:00:01:56] ENGINE Bus STARTED 2024-09-17T00:01:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:58 smithi194 ceph-mon[25713]: pgmap v5: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:01:58.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:58 smithi194 ceph-mon[25713]: mgrmap e33: smithi007.czdphd(active, since 5s) 2024-09-17T00:01:59.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 101836 10.46 MB/sec execute 103 sec latency 40.232 ms 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.755 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:01:59 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/4080911852' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:01:59.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:01:59 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:00.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 102126 10.45 MB/sec execute 104 sec latency 69.422 ms 2024-09-17T00:02:00.764 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:00 smithi007 ceph-mon[23721]: pgmap v6: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:02:00.764 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:00 smithi007 ceph-mon[23721]: mgrmap e34: smithi007.czdphd(active, since 7s), standbys: smithi194.xlteft 2024-09-17T00:02:00.764 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:00 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-17T00:02:00.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:00 smithi194 ceph-mon[25713]: pgmap v6: 97 pgs: 97 active+clean; 342 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-17T00:02:00.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:00 smithi194 ceph-mon[25713]: mgrmap e34: smithi007.czdphd(active, since 7s), standbys: smithi194.xlteft 2024-09-17T00:02:00.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:00 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-17T00:02:01.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 102547 10.53 MB/sec execute 105 sec latency 53.512 ms 2024-09-17T00:02:02.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 102972 10.57 MB/sec execute 106 sec latency 50.042 ms 2024-09-17T00:02:02.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:02 smithi007 ceph-mon[23721]: pgmap v7: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 MiB/s wr, 389 op/s 2024-09-17T00:02:02.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:02 smithi194 ceph-mon[25713]: pgmap v7: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 MiB/s wr, 389 op/s 2024-09-17T00:02:03.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 103272 10.57 MB/sec execute 107 sec latency 69.167 ms 2024-09-17T00:02:04.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 103506 10.53 MB/sec execute 108 sec latency 39.766 ms 2024-09-17T00:02:04.568 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:04 smithi007 ceph-mon[23721]: pgmap v8: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.4 MiB/s wr, 300 op/s 2024-09-17T00:02:04.568 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:04 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.568 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:04 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.568 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:04 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.568 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:04 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:04 smithi194 ceph-mon[25713]: pgmap v8: 97 pgs: 97 active+clean; 335 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 2.4 MiB/s wr, 300 op/s 2024-09-17T00:02:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:04 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:04 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:04 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:04.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:04 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:05.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 103791 10.45 MB/sec execute 109 sec latency 57.948 ms 2024-09-17T00:02:05.308 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:02:06.094 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:06 smithi007 ceph-mon[23721]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:06.095 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:06 smithi007 ceph-mon[23721]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-09-17T00:02:06.095 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:06 smithi007 ceph-mon[23721]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:06.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 104098 10.38 MB/sec execute 110 sec latency 39.883 ms 2024-09-17T00:02:06.336 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:02:06.336 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (82s) 2s ago 10m 28.1M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:02:06.336 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (10m) 2s ago 10m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:02:06.336 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (9m) 7s ago 9m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:02:06.336 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (75s) 2s ago 10m 47.6M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (4m) 2s ago 4m 36.1M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (4m) 7s ago 4m 13.6M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (30s) 2s ago 11m 474M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283 running (2m) 7s ago 9m 137M - 19.1.1-330-gc2ddf972 ebb0615149de 6d63d1dc2865 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (11m) 2s ago 11m 55.4M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (9m) 7s ago 9m 43.0M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (4m) 2s ago 4m 256M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (3m) 7s ago 3m 85.7M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (68s) 2s ago 10m 8274k - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (41s) 7s ago 9m 7838k - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:02:06.337 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (8m) 2s ago 8m 401M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (8m) 2s ago 8m 302M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (7m) 2s ago 7m 272M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (7m) 2s ago 7m 358M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (7m) 7s ago 7m 353M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (6m) 7s ago 6m 393M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (6m) 7s ago 6m 339M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (5m) 7s ago 5m 455M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:02:06.338 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (46s) 2s ago 9m 48.1M - 2.43.0 a07b618ecd1d 492bbf59541b 2024-09-17T00:02:06.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:06 smithi194 ceph-mon[25713]: from='client.14780 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:06.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:06 smithi194 ceph-mon[25713]: pgmap v9: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 464 op/s 2024-09-17T00:02:06.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:06 smithi194 ceph-mon[25713]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:02:07.068 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:02:07.069 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:02:07.221 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:07 smithi007 ceph-mon[23721]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:07.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 104298 10.33 MB/sec execute 111 sec latency 75.054 ms 2024-09-17T00:02:07.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:07 smithi194 ceph-mon[25713]: from='client.14788 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:07.603 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:02:07.603 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T00:02:07.604 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "", 2024-09-17T00:02:07.605 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:02:07.605 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:02:08.244 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:02:08.245 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:08 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/535093798' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:02:08.245 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:08 smithi007 ceph-mon[23721]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-17T00:02:08.245 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:08 smithi007 ceph-mon[23721]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:08.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 104635 10.37 MB/sec execute 112 sec latency 53.240 ms 2024-09-17T00:02:08.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:08 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/535093798' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:02:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:08 smithi194 ceph-mon[25713]: pgmap v10: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-17T00:02:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:08 smithi194 ceph-mon[25713]: from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:09.269 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:09 smithi007 ceph-mon[23721]: from='client.? 172.21.15.7:0/2262451737' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:02:09.269 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:09 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:09.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 105057 10.34 MB/sec execute 113 sec latency 61.665 ms 2024-09-17T00:02:09.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:09 smithi194 ceph-mon[25713]: from='client.? 172.21.15.7:0/2262451737' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:02:09.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:09 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:10.276 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:10 smithi007 ceph-mon[23721]: pgmap v11: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-17T00:02:10.278 INFO:teuthology.orchestra.run.smithi007.stdout: 5 105521 10.33 MB/sec execute 114 sec latency 74.508 ms 2024-09-17T00:02:10.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:10 smithi194 ceph-mon[25713]: pgmap v11: 97 pgs: 97 active+clean; 341 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 MiB/s wr, 415 op/s 2024-09-17T00:02:11.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 105832 10.35 MB/sec execute 115 sec latency 56.333 ms 2024-09-17T00:02:12.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 106214 10.39 MB/sec execute 116 sec latency 58.835 ms 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 MiB/s wr, 611 op/s 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:02:12.345 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:02:12.346 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:12 smithi007 ceph-mon[23721]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: pgmap v12: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 5.8 MiB/s wr, 611 op/s 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:02:12.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:02:12.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:02:12.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:12 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:02:13.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 106564 10.40 MB/sec execute 117 sec latency 63.529 ms 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: Reconfiguring prometheus.smithi007 (dependencies changed)... 2024-09-17T00:02:13.372 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:13 smithi007 ceph-mon[23721]: Reconfiguring daemon prometheus.smithi007 on smithi007 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: Reconfiguring prometheus.smithi007 (dependencies changed)... 2024-09-17T00:02:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:13 smithi194 ceph-mon[25713]: Reconfiguring daemon prometheus.smithi007 on smithi007 2024-09-17T00:02:14.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 106828 10.40 MB/sec execute 118 sec latency 57.241 ms 2024-09-17T00:02:14.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:14 smithi194 ceph-mon[25713]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-17T00:02:14.682 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:14 smithi007 ceph-mon[23721]: pgmap v13: 97 pgs: 97 active+clean; 334 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-17T00:02:15.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 107001 10.34 MB/sec execute 119 sec latency 94.720 ms 2024-09-17T00:02:16.218 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:15 smithi007 ceph-mon[23721]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-09-17T00:02:16.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 107320 10.29 MB/sec execute 120 sec latency 125.644 ms 2024-09-17T00:02:16.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:15 smithi194 ceph-mon[25713]: pgmap v14: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 590 op/s 2024-09-17T00:02:17.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 107677 10.25 MB/sec execute 121 sec latency 43.326 ms 2024-09-17T00:02:18.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 108005 10.25 MB/sec execute 122 sec latency 43.925 ms 2024-09-17T00:02:18.518 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:18 smithi007 ceph-mon[23721]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-17T00:02:18.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:18 smithi194 ceph-mon[25713]: pgmap v15: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-17T00:02:19.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 108502 10.26 MB/sec execute 123 sec latency 59.320 ms 2024-09-17T00:02:20.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 108921 10.23 MB/sec execute 124 sec latency 48.207 ms 2024-09-17T00:02:20.583 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:20 smithi007 ceph-mon[23721]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-17T00:02:20.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:20 smithi194 ceph-mon[25713]: pgmap v16: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 393 op/s 2024-09-17T00:02:21.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 109189 10.21 MB/sec execute 125 sec latency 67.897 ms 2024-09-17T00:02:22.130 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:22 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:22.131 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:22 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:22.131 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:22 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:02:22.131 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:22 smithi007 ceph-mon[23721]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:02:22.131 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:22 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:22.131 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:22 smithi007 ceph-mon[23721]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 611 op/s 2024-09-17T00:02:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:22 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:22 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:22 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:02:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:22 smithi194 ceph-mon[25713]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-17T00:02:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:22 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:22 smithi194 ceph-mon[25713]: pgmap v17: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.7 MiB/s wr, 611 op/s 2024-09-17T00:02:22.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 109513 10.25 MB/sec execute 126 sec latency 81.145 ms 2024-09-17T00:02:23.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:23 smithi007 ceph-mon[23721]: Upgrade: Updating mgr.smithi194.xlteft 2024-09-17T00:02:23.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:23 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:02:23.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:23 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:02:23.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:23 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:23.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:23 smithi007 ceph-mon[23721]: Deploying daemon mgr.smithi194.xlteft on smithi194 2024-09-17T00:02:23.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 109870 10.26 MB/sec execute 127 sec latency 37.265 ms 2024-09-17T00:02:23.375 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:23 smithi194 ceph-mon[25713]: Upgrade: Updating mgr.smithi194.xlteft 2024-09-17T00:02:23.375 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:23 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:02:23.375 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:23 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:02:23.375 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:23 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:23.375 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:23 smithi194 ceph-mon[25713]: Deploying daemon mgr.smithi194.xlteft on smithi194 2024-09-17T00:02:24.180 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:24 smithi007 ceph-mon[23721]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-17T00:02:24.180 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:24 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:24.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 110211 10.28 MB/sec execute 128 sec latency 61.353 ms 2024-09-17T00:02:24.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:24 smithi194 ceph-mon[25713]: pgmap v18: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-17T00:02:24.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:24 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:25.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 110432 10.27 MB/sec execute 129 sec latency 53.877 ms 2024-09-17T00:02:25.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:25 smithi007 ceph-mon[23721]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 604 op/s 2024-09-17T00:02:26.147 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:25 smithi194 ceph-mon[25713]: pgmap v19: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.6 MiB/s wr, 604 op/s 2024-09-17T00:02:26.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 110694 10.22 MB/sec execute 130 sec latency 48.795 ms 2024-09-17T00:02:27.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 110997 10.16 MB/sec execute 131 sec latency 61.666 ms 2024-09-17T00:02:27.530 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:27 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.530 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:27 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.530 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:27 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:27.530 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:27 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.530 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:27 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.773 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:27 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.774 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:27 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.774 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:27 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:27.774 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:27 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:27.774 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:27 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:28.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 111274 10.12 MB/sec execute 132 sec latency 54.097 ms 2024-09-17T00:02:28.940 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:28 smithi007 ceph-mon[23721]: pgmap v20: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-09-17T00:02:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:28 smithi194 ceph-mon[25713]: pgmap v20: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-09-17T00:02:29.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 111556 10.12 MB/sec execute 133 sec latency 67.887 ms 2024-09-17T00:02:30.279 INFO:teuthology.orchestra.run.smithi007.stdout: 5 112005 10.13 MB/sec execute 134 sec latency 49.153 ms 2024-09-17T00:02:31.010 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:30 smithi194 ceph-mon[25713]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-09-17T00:02:31.010 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:30 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:31.010 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:30 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:31.023 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:30 smithi007 ceph-mon[23721]: pgmap v21: 97 pgs: 97 active+clean; 342 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.1 MiB/s wr, 407 op/s 2024-09-17T00:02:31.023 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:30 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:31.023 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:30 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:31.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 112467 10.10 MB/sec execute 135 sec latency 60.290 ms 2024-09-17T00:02:32.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 112734 10.09 MB/sec execute 136 sec latency 32.465 ms 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 603 op/s 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: Standby manager daemon smithi194.xlteft restarted 2024-09-17T00:02:32.807 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:32 smithi007 ceph-mon[23721]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:02:32.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: pgmap v22: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 5.8 MiB/s wr, 603 op/s 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1613229398' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft restarted 2024-09-17T00:02:32.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:32 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:02:33.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 113058 10.13 MB/sec execute 137 sec latency 63.881 ms 2024-09-17T00:02:34.097 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:33 smithi007 ceph-mon[23721]: mgrmap e35: smithi007.czdphd(active, since 40s), standbys: smithi194.xlteft 2024-09-17T00:02:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:33 smithi194 ceph-mon[25713]: mgrmap e35: smithi007.czdphd(active, since 40s), standbys: smithi194.xlteft 2024-09-17T00:02:34.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 113355 10.13 MB/sec execute 138 sec latency 81.294 ms 2024-09-17T00:02:35.093 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:34 smithi194 ceph-mon[25713]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 385 op/s 2024-09-17T00:02:35.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:34 smithi007 ceph-mon[23721]: pgmap v23: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 385 op/s 2024-09-17T00:02:35.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 113749 10.16 MB/sec execute 139 sec latency 48.222 ms 2024-09-17T00:02:36.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 113969 10.14 MB/sec execute 140 sec latency 78.892 ms 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:02:36.402 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: Upgrade: Setting container_image for all mgr 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi007.czdphd"}]: dispatch 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi007.czdphd"}]': finished 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.xlteft"}]: dispatch 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.xlteft"}]': finished 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T00:02:36.403 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:36 smithi007 ceph-mon[23721]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 584 op/s 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:36.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: Upgrade: Setting container_image for all mgr 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi007.czdphd"}]: dispatch 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi007.czdphd"}]': finished 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.xlteft"}]: dispatch 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi194.xlteft"}]': finished 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T00:02:36.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:36 smithi194 ceph-mon[25713]: pgmap v24: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.4 MiB/s wr, 584 op/s 2024-09-17T00:02:37.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 114224 10.10 MB/sec execute 141 sec latency 52.153 ms 2024-09-17T00:02:38.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 114614 10.05 MB/sec execute 142 sec latency 40.672 ms 2024-09-17T00:02:38.697 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:38 smithi007 ceph-mon[23721]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-17T00:02:38.697 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:38 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:38.697 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:38 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:02:38.697 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:38 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:02:38.697 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:38 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:38.782 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:02:38.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:38 smithi194 ceph-mon[25713]: pgmap v25: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-17T00:02:38.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:38 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:38.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:38 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:02:38.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:38 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:02:38.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:38 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:02:39.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 115007 10.05 MB/sec execute 143 sec latency 74.035 ms 2024-09-17T00:02:39.712 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:39 smithi007 ceph-mon[23721]: Upgrade: Updating mon.smithi007 2024-09-17T00:02:39.712 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:39 smithi007 ceph-mon[23721]: Deploying daemon mon.smithi007 on smithi007 2024-09-17T00:02:39.712 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:39 smithi007 ceph-mon[23721]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:39.713 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:39 smithi007 ceph-mon[23721]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:39 smithi194 ceph-mon[25713]: Upgrade: Updating mon.smithi007 2024-09-17T00:02:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:39 smithi194 ceph-mon[25713]: Deploying daemon mon.smithi007 on smithi007 2024-09-17T00:02:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:39 smithi194 ceph-mon[25713]: from='client.24589 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:39 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (115s) 4s ago 10m 28.1M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (10m) 4s ago 10m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (10m) 9s ago 10m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (109s) 4s ago 10m 46.4M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (4m) 4s ago 4m 36.0M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (4m) 9s ago 4m 13.8M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:02:39.943 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (64s) 4s ago 12m 492M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (14s) 9s ago 10m 93.9M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (12m) 4s ago 12m 54.2M 2048M 17.2.0 e1d6a67b021e 97f51479929a 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (10m) 9s ago 10m 40.2M 2048M 17.2.0 e1d6a67b021e 32c4fa431c9c 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (4m) 4s ago 4m 248M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (4m) 9s ago 4m 85.8M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (101s) 4s ago 10m 7796k - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (74s) 9s ago 10m 8154k - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (9m) 4s ago 9m 435M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (8m) 4s ago 8m 333M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (8m) 4s ago 8m 313M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (8m) 4s ago 8m 389M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (7m) 9s ago 7m 379M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:02:39.944 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (7m) 9s ago 7m 422M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:02:39.945 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (6m) 9s ago 6m 374M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:02:39.945 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (6m) 9s ago 6m 476M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:02:39.945 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (19s) 4s ago 9m 45.9M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:02:40.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 115491 10.08 MB/sec execute 144 sec latency 40.055 ms 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:02:40.621 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:02:40.622 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:02:40.718 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:40 smithi007 ceph-mon[23721]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:40.718 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:40 smithi007 ceph-mon[23721]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-17T00:02:40.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:40 smithi194 ceph-mon[25713]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:02:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:40 smithi194 ceph-mon[25713]: pgmap v26: 97 pgs: 97 active+clean; 337 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-17T00:02:41.211 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:02:41.211 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:02:41.211 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "2/23 daemons upgraded", 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:02:41.212 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:02:41.223 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:41 smithi007 systemd[1]: Stopping Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:02:41.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 115953 10.06 MB/sec execute 145 sec latency 55.301 ms 2024-09-17T00:02:41.474 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:41 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[23717]: 2024-09-17T00:02:41.400+0000 7fcba4680700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi007 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:02:41.474 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:41 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[23717]: 2024-09-17T00:02:41.400+0000 7fcba4680700 -1 mon.smithi007@0(leader) e2 *** Got Signal Terminated *** 2024-09-17T00:02:41.725 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:41 smithi007 podman[65294]: 2024-09-17 00:02:41.520259951 +0000 UTC m=+0.254560369 container died 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, io.openshift.tags=base centos centos-stream, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, distribution-scope=public, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8, ceph=True) 2024-09-17T00:02:41.863 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:02:42.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 116267 10.05 MB/sec execute 146 sec latency 40.623 ms 2024-09-17T00:02:42.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:42 smithi007 podman[65294]: 2024-09-17 00:02:42.588072595 +0000 UTC m=+1.322373008 container cleanup 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., architecture=x86_64, name=centos-stream, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, distribution-scope=public, ceph=True, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream) 2024-09-17T00:02:42.782 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:42 smithi007 bash[65294]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007 2024-09-17T00:02:43.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 116612 10.09 MB/sec execute 147 sec latency 62.546 ms 2024-09-17T00:02:43.297 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:43 smithi007 podman[65337]: 2024-09-17 00:02:43.24098755 +0000 UTC m=+1.714560254 container remove 97f51479929afe77553d0ed0f287364433166d01d7888731c5c17fe6914f0aff (image=quay.io/ceph/ceph:v17.2.0, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, build-date=2022-05-03T08:36:31.336870, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, ceph=True, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, io.k8s.display-name=CentOS Stream 8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.openshift.expose-services=, maintainer=Guillaume Abrioux , distribution-scope=public, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-17T00:02:43.819 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:43 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service: Deactivated successfully. 2024-09-17T00:02:43.819 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:43 smithi007 systemd[1]: Stopped Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:02:43.819 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:43 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service: Consumed 18.093s CPU time. 2024-09-17T00:02:44.087 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:43 smithi007 systemd[1]: Starting Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:02:44.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 116911 10.09 MB/sec execute 148 sec latency 80.705 ms 2024-09-17T00:02:44.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 podman[65469]: 2024-09-17 00:02:44.032288276 +0000 UTC m=+0.021778951 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:02:44.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 podman[65469]: 2024-09-17 00:02:44.191891408 +0000 UTC m=+0.181382073 container create b047b28b4edac807811478889d46e09a66d18bd77a524e9e006fe6b656bcf349 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True) 2024-09-17T00:02:44.602 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 podman[65469]: 2024-09-17 00:02:44.544836155 +0000 UTC m=+0.534326832 container init b047b28b4edac807811478889d46e09a66d18bd77a524e9e006fe6b656bcf349 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-17T00:02:44.602 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 podman[65469]: 2024-09-17 00:02:44.552851039 +0000 UTC m=+0.542341702 container start b047b28b4edac807811478889d46e09a66d18bd77a524e9e006fe6b656bcf349 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:02:44.602 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T00:02:44.602 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T00:02:44.602 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: pidfile_write: ignore empty --pid-file 2024-09-17T00:02:44.602 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: load: jerasure load: lrc 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: RocksDB version: 7.9.2 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Git sha 0 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: DB SUMMARY 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: DB Session ID: 5WD5ZI8L4ACLT0SCYPWV 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: CURRENT file: CURRENT 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: MANIFEST file: MANIFEST-000015 size: 979 Bytes 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi007/store.db dir, Total Num: 1, files: 000030.sst 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi007/store.db: 000028.log size: 1490784 ; 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.error_if_exists: 0 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.create_if_missing: 0 2024-09-17T00:02:44.603 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.paranoid_checks: 1 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.env: 0x55d6250be940 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.info_log: 0x55d6275cf8c0 2024-09-17T00:02:44.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.statistics: (nil) 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.use_fsync: 0 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_log_file_size: 0 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.allow_fallocate: 1 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T00:02:44.605 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.use_direct_reads: 0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.db_log_dir: 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.wal_dir: 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.write_buffer_manager: 0x55d6275dedc0 2024-09-17T00:02:44.606 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.unordered_write: 0 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T00:02:44.607 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.row_cache: None 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.wal_filter: None 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.two_write_queues: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.wal_compression: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.atomic_flush: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.log_readahead_size: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T00:02:44.608 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_background_jobs: 2 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_background_compactions: -1 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_subcompactions: 1 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T00:02:44.609 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_open_files: -1 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_background_flushes: -1 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Compression algorithms supported: 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kZSTD supported: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kXpressCompression supported: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kBZip2Compression supported: 0 2024-09-17T00:02:44.610 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T00:02:44.611 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kLZ4Compression supported: 1 2024-09-17T00:02:44.611 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kZlibCompression supported: 1 2024-09-17T00:02:44.611 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T00:02:44.612 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: kSnappyCompression supported: 1 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi007/store.db/MANIFEST-000015 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.merge_operator: 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_filter: None 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d6275cf520) 2024-09-17T00:02:44.613 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: cache_index_and_filter_blocks: 1 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: pin_top_level_index_and_filter: 1 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: index_type: 0 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: data_block_index_type: 0 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: index_shortening: 1 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: checksum: 4 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: no_block_cache: 0 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache: 0x55d6275c31f0 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache_name: BinnedLRUCache 2024-09-17T00:02:44.614 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache_options: 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: capacity : 536870912 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: num_shard_bits : 4 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: strict_capacity_limit : 0 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: high_pri_pool_ratio: 0.000 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_cache_compressed: (nil) 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: persistent_cache: (nil) 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_size: 4096 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_size_deviation: 10 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_restart_interval: 16 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: index_block_restart_interval: 1 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: metadata_block_size: 4096 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: partition_filters: 0 2024-09-17T00:02:44.615 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: use_delta_encoding: 1 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: filter_policy: bloomfilter 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: whole_key_filtering: 1 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: verify_compression: 0 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: read_amp_bytes_per_bit: 0 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: format_version: 5 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: enable_index_compression: 1 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: block_align: 0 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: max_auto_readahead_size: 262144 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: prepopulate_block_cache: 0 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: initial_auto_readahead_size: 8192 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T00:02:44.616 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression: NoCompression 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.num_levels: 7 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T00:02:44.617 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T00:02:44.618 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T00:02:44.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T00:02:44.620 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.inplace_update_support: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.bloom_locality: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.max_successive_merges: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T00:02:44.621 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.ttl: 2592000 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.enable_blob_files: false 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.min_blob_size: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T00:02:44.622 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi007/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11921, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: eed1488e-57c2-44bf-bd4e-1b7e71b80108 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726531364584595, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726531364591118, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 1326063, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11915, "largest_seqno": 12705, "table_properties": {"data_size": 1321806, "index_size": 2321, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 7838, "raw_average_key_size": 23, "raw_value_size": 1314348, "raw_average_value_size": 3911, "num_data_blocks": 106, "num_entries": 336, "num_filter_entries": 336, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726531364, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "eed1488e-57c2-44bf-bd4e-1b7e71b80108", "db_session_id": "5WD5ZI8L4ACLT0SCYPWV", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726531364591240, "job": 1, "event": "recovery_finished"} 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi007/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d6275f0e00 2024-09-17T00:02:44.623 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: DB pointer 0x55d6276ec000 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** DB Stats ** 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.624 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** Compaction Stats [default] ** 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: L0 1/0 1.26 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 342.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: L6 1/0 7.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Sum 2/0 8.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 342.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 342.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** Compaction Stats [default] ** 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 342.6 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Cumulative compaction: 0.00 GB write, 132.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Interval compaction: 0.00 GB write, 132.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Block cache BinnedLRUCache@0x55d6275c31f0#2 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.626 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: starting mon.smithi007 rank 0 at public addrs [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] at bind addrs [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi007 fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???) e2 preinit fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).mds e7 new map 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).mds e7 print_map 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: e7 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: legacy client fscid: 1 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Filesystem 'foofs' (1) 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: fs_name foofs 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: epoch 5 2024-09-17T00:02:44.627 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: created 2024-09-16T23:57:36.932009+0000 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: modified 2024-09-16T23:57:43.146036+0000 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: tableserver 0 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: root 0 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: session_timeout 60 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: session_autoclose 300 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: max_file_size 1099511627776 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: max_xattr_size 65536 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: required_client_features {} 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: last_failure 0 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: last_failure_osd_epoch 0 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T00:02:44.628 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: max_mds 1 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: in 0 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: up {0=14476} 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: failed 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: damaged 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: stopped 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: data_pools [3] 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: metadata_pool 2 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: inline_data disabled 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: balancer 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: bal_rank_mask -1 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: standby_count_wanted 1 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: qdb_cluster leader: 0 members: 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: [mds.foofs.smithi007.fkkfbl{0:14476} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.7:6834/4061963992,v1:172.21.15.7:6835/4061963992] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T00:02:44.629 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Standby daemons: 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: [mds.foofs.smithi194.emredf{-1:24313} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).osd e71 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).osd e71 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T00:02:44.630 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 ceph-mon[65487]: mon.smithi007@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-17T00:02:44.853 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 bash[65469]: b047b28b4edac807811478889d46e09a66d18bd77a524e9e006fe6b656bcf349 2024-09-17T00:02:44.853 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:44 smithi007 systemd[1]: Started Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:02:45.280 INFO:teuthology.orchestra.run.smithi007.stdout: 5 117266 10.11 MB/sec execute 149 sec latency 40.646 ms 2024-09-17T00:02:46.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 117337 10.07 MB/sec execute 150 sec latency 713.765 ms 2024-09-17T00:02:47.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 117337 10.00 MB/sec execute 151 sec latency 1713.814 ms 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: from='client.14822 172.21.15.7:0/2334703201' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.6 MiB/s wr, 623 op/s 2024-09-17T00:02:47.879 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: mon.smithi007 calling monitor election 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: mon.smithi007 is new leader, mons smithi007,smithi194 in quorum (ranks 0,1) 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: monmap epoch 2 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: last_changed 2024-09-16T23:52:27.691419+0000 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: created 2024-09-16T23:50:21.589767+0000 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: min_mon_release 17 (quincy) 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: election_strategy: 1 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: 0: [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] mon.smithi007 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: mgrmap e35: smithi007.czdphd(active, since 55s), standbys: smithi194.xlteft 2024-09-17T00:02:47.880 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: overall HEALTH_OK 2024-09-17T00:02:47.881 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: from='mgr.14752 ' entity='' 2024-09-17T00:02:47.881 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:47 smithi007 ceph-mon[65487]: mgrmap e36: smithi007.czdphd(active, since 55s), standbys: smithi194.xlteft 2024-09-17T00:02:48.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:02:48.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: from='mgr.14752 172.21.15.7:0/3351175226' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:02:48.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: pgmap v27: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.0 MiB/s wr, 610 op/s 2024-09-17T00:02:48.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: from='client.14822 172.21.15.7:0/2334703201' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:02:48.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: pgmap v28: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 414 op/s 2024-09-17T00:02:48.065 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: pgmap v29: 97 pgs: 97 active+clean; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.3 KiB/s rd, 6.6 MiB/s wr, 623 op/s 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: mon.smithi007 calling monitor election 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: mon.smithi007 is new leader, mons smithi007,smithi194 in quorum (ranks 0,1) 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: monmap epoch 2 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: last_changed 2024-09-16T23:52:27.691419+0000 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: created 2024-09-16T23:50:21.589767+0000 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: min_mon_release 17 (quincy) 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: election_strategy: 1 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: 0: [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] mon.smithi007 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:02:48.066 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: osdmap e71: 8 total, 8 up, 8 in 2024-09-17T00:02:48.067 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: mgrmap e35: smithi007.czdphd(active, since 55s), standbys: smithi194.xlteft 2024-09-17T00:02:48.067 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: overall HEALTH_OK 2024-09-17T00:02:48.067 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: from='mgr.14752 ' entity='' 2024-09-17T00:02:48.067 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:47 smithi194 ceph-mon[25713]: mgrmap e36: smithi007.czdphd(active, since 55s), standbys: smithi194.xlteft 2024-09-17T00:02:48.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 117522 9.96 MB/sec execute 152 sec latency 2051.660 ms 2024-09-17T00:02:49.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 117899 9.94 MB/sec execute 153 sec latency 44.432 ms 2024-09-17T00:02:49.924 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:49 smithi007 ceph-mon[65487]: mgrmap e37: smithi007.czdphd(active, since 56s), standbys: smithi194.xlteft 2024-09-17T00:02:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:49 smithi194 ceph-mon[25713]: mgrmap e37: smithi007.czdphd(active, since 56s), standbys: smithi194.xlteft 2024-09-17T00:02:50.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 118264 9.90 MB/sec execute 154 sec latency 45.705 ms 2024-09-17T00:02:51.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 118710 9.94 MB/sec execute 155 sec latency 35.177 ms 2024-09-17T00:02:52.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 119341 9.94 MB/sec execute 156 sec latency 39.497 ms 2024-09-17T00:02:53.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 119636 9.92 MB/sec execute 157 sec latency 127.590 ms 2024-09-17T00:02:53.488 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:53 smithi007 ceph-mon[65487]: Standby manager daemon smithi194.xlteft restarted 2024-09-17T00:02:53.488 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:53 smithi007 ceph-mon[65487]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:02:53.488 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:53 smithi007 ceph-mon[65487]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:02:53.488 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:53 smithi007 ceph-mon[65487]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:02:53.488 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:53 smithi007 ceph-mon[65487]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:02:53.488 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:53 smithi007 ceph-mon[65487]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:02:53.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:53 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft restarted 2024-09-17T00:02:53.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:53 smithi194 ceph-mon[25713]: Standby manager daemon smithi194.xlteft started 2024-09-17T00:02:53.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:53 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/crt"}]: dispatch 2024-09-17T00:02:53.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:53 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-17T00:02:53.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:53 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi194.xlteft/key"}]: dispatch 2024-09-17T00:02:53.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:53 smithi194 ceph-mon[25713]: from='mgr.? 172.21.15.194:0/1325135480' entity='mgr.smithi194.xlteft' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-17T00:02:54.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 120052 9.97 MB/sec execute 158 sec latency 78.885 ms 2024-09-17T00:02:54.497 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:54 smithi007 ceph-mon[65487]: mgrmap e38: smithi007.czdphd(active, since 60s), standbys: smithi194.xlteft 2024-09-17T00:02:54.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:54 smithi194 ceph-mon[25713]: mgrmap e38: smithi007.czdphd(active, since 60s), standbys: smithi194.xlteft 2024-09-17T00:02:55.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 120414 9.99 MB/sec execute 159 sec latency 60.978 ms 2024-09-17T00:02:55.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: Active manager daemon smithi007.czdphd restarted 2024-09-17T00:02:55.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: Activating manager daemon smithi007.czdphd 2024-09-17T00:02:55.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: mgrmap e39: smithi007.czdphd(active, starting, since 0.0329796s), standbys: smithi194.xlteft 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:02:55.513 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: Manager daemon smithi007.czdphd is now available 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:02:55.514 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-17T00:02:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: Active manager daemon smithi007.czdphd restarted 2024-09-17T00:02:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: Activating manager daemon smithi007.czdphd 2024-09-17T00:02:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T00:02:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: mgrmap e39: smithi007.czdphd(active, starting, since 0.0329796s), standbys: smithi194.xlteft 2024-09-17T00:02:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:02:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi007.czdphd", "id": "smithi007.czdphd"}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr metadata", "who": "smithi194.xlteft", "id": "smithi194.xlteft"}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:02:55.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: Manager daemon smithi007.czdphd is now available 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/mirror_snapshot_schedule"}]: dispatch 2024-09-17T00:02:55.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:55 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi007.czdphd/trash_purge_schedule"}]: dispatch 2024-09-17T00:02:56.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 120797 10.01 MB/sec execute 160 sec latency 53.416 ms 2024-09-17T00:02:56.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:56 smithi007 ceph-mon[65487]: mgrmap e40: smithi007.czdphd(active, since 1.03768s), standbys: smithi194.xlteft 2024-09-17T00:02:56.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:56 smithi007 ceph-mon[65487]: pgmap v3: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:02:56.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:56 smithi194 ceph-mon[25713]: mgrmap e40: smithi007.czdphd(active, since 1.03768s), standbys: smithi194.xlteft 2024-09-17T00:02:56.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:56 smithi194 ceph-mon[25713]: pgmap v3: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:02:57.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 121003 9.99 MB/sec execute 161 sec latency 43.451 ms 2024-09-17T00:02:57.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: [17/Sep/2024:00:02:56] ENGINE Bus STARTING 2024-09-17T00:02:57.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: [17/Sep/2024:00:02:56] ENGINE Serving on http://172.21.15.7:8765 2024-09-17T00:02:57.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: pgmap v4: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:02:57.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: [17/Sep/2024:00:02:56] ENGINE Serving on https://172.21.15.7:7150 2024-09-17T00:02:57.808 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: [17/Sep/2024:00:02:56] ENGINE Bus STARTED 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: [17/Sep/2024:00:02:56] ENGINE Error in HTTPServer.serve 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: Traceback (most recent call last): 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._connections.run(self.expiration_interval) 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._run(expiration_interval) 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: s = self.context.wrap_socket( 2024-09-17T00:02:57.809 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: return self.sslsocket_class._create( 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self.do_handshake() 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: self._sslobj.do_handshake() 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T00:02:57.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:57 smithi007 ceph-mon[65487]: mgrmap e41: smithi007.czdphd(active, since 2s), standbys: smithi194.xlteft 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:02:56] ENGINE Bus STARTING 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:02:56] ENGINE Serving on http://172.21.15.7:8765 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: pgmap v4: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:02:56] ENGINE Serving on https://172.21.15.7:7150 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:02:56] ENGINE Bus STARTED 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: [17/Sep/2024:00:02:56] ENGINE Error in HTTPServer.serve 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Traceback (most recent call last): 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._connections.run(self.expiration_interval) 2024-09-17T00:02:57.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._run(expiration_interval) 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: s = self.context.wrap_socket( 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: return self.sslsocket_class._create( 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self.do_handshake() 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: self._sslobj.do_handshake() 2024-09-17T00:02:57.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-17T00:02:57.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:57 smithi194 ceph-mon[25713]: mgrmap e41: smithi007.czdphd(active, since 2s), standbys: smithi194.xlteft 2024-09-17T00:02:58.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 121197 9.95 MB/sec execute 162 sec latency 69.180 ms 2024-09-17T00:02:59.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 121449 9.92 MB/sec execute 163 sec latency 118.680 ms 2024-09-17T00:02:59.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:59 smithi007 ceph-mon[65487]: pgmap v5: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:02:59.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:59 smithi007 ceph-mon[65487]: mgrmap e42: smithi007.czdphd(active, since 4s), standbys: smithi194.xlteft 2024-09-17T00:02:59.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:02:59.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:02:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:00.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:59 smithi194 ceph-mon[25713]: pgmap v5: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:03:00.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:59 smithi194 ceph-mon[25713]: mgrmap e42: smithi007.czdphd(active, since 4s), standbys: smithi194.xlteft 2024-09-17T00:03:00.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:59 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:00.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:02:59 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:00.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 121666 9.87 MB/sec execute 164 sec latency 67.229 ms 2024-09-17T00:03:00.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:00.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:00.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:00 smithi007 ceph-mon[65487]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:03:01.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 121973 9.84 MB/sec execute 165 sec latency 41.751 ms 2024-09-17T00:03:01.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:00 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:01.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:00 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:01.392 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:00 smithi194 ceph-mon[25713]: pgmap v6: 97 pgs: 97 active+clean; 341 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-17T00:03:02.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 122311 9.85 MB/sec execute 166 sec latency 51.600 ms 2024-09-17T00:03:03.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 122770 9.84 MB/sec execute 167 sec latency 62.044 ms 2024-09-17T00:03:03.711 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:03 smithi007 ceph-mon[65487]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 2.6 MiB/s wr, 306 op/s 2024-09-17T00:03:03.711 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:03 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:03.711 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:03 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:03.783 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:03 smithi194 ceph-mon[25713]: pgmap v7: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 37 KiB/s rd, 2.6 MiB/s wr, 306 op/s 2024-09-17T00:03:03.783 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:03 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:03.783 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:03 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:04.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 123094 9.83 MB/sec execute 168 sec latency 62.992 ms 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-17T00:03:04.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:03:04.983 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:03:04.983 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 237 op/s 2024-09-17T00:03:04.983 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:04 smithi007 ceph-mon[65487]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.conf 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.conf 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-17T00:03:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: pgmap v8: 97 pgs: 97 active+clean; 332 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 28 KiB/s rd, 2.0 MiB/s wr, 237 op/s 2024-09-17T00:03:05.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:04 smithi194 ceph-mon[25713]: Updating smithi194:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:03:05.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 123395 9.83 MB/sec execute 169 sec latency 50.291 ms 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.260 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.261 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.261 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:03:06.261 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:06.261 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T00:03:06.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 123745 9.86 MB/sec execute 170 sec latency 45.795 ms 2024-09-17T00:03:06.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: Updating smithi007:/etc/ceph/ceph.client.admin.keyring 2024-09-17T00:03:06.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: Updating smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: Updating smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:06.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:06 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-17T00:03:07.279 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:07 smithi007 ceph-mon[65487]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 422 op/s 2024-09-17T00:03:07.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 124151 9.89 MB/sec execute 171 sec latency 55.878 ms 2024-09-17T00:03:07.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:07 smithi194 ceph-mon[25713]: pgmap v9: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 422 op/s 2024-09-17T00:03:08.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 124438 9.89 MB/sec execute 172 sec latency 50.297 ms 2024-09-17T00:03:08.311 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:08 smithi007 ceph-mon[65487]: Upgrade: Updating mon.smithi194 2024-09-17T00:03:08.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:08.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:03:08.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:03:08.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:08.312 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:08 smithi007 ceph-mon[65487]: Deploying daemon mon.smithi194 on smithi194 2024-09-17T00:03:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:08 smithi194 ceph-mon[25713]: Upgrade: Updating mon.smithi194 2024-09-17T00:03:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:08 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:08 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:03:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:08 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:03:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:08 smithi194 ceph-mon[25713]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:08.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:08 smithi194 ceph-mon[25713]: Deploying daemon mon.smithi194 on smithi194 2024-09-17T00:03:09.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 124667 9.87 MB/sec execute 173 sec latency 72.223 ms 2024-09-17T00:03:09.339 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:09 smithi007 ceph-mon[65487]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.5 MiB/s wr, 383 op/s 2024-09-17T00:03:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:09 smithi194 systemd[1]: Stopping Ceph mon.smithi194 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:03:09.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:09 smithi194 ceph-mon[25713]: pgmap v10: 97 pgs: 97 active+clean; 336 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 3.5 MiB/s wr, 383 op/s 2024-09-17T00:03:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194[25700]: 2024-09-17T00:03:09.840+0000 7ff7a05be700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi194 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:03:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194[25700]: 2024-09-17T00:03:09.840+0000 7ff7a05be700 -1 mon.smithi194@1(peon) e2 *** Got Signal Terminated *** 2024-09-17T00:03:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:09 smithi194 podman[51367]: 2024-09-17 00:03:09.993392378 +0000 UTC m=+0.603027288 container died 32c4fa431c9c102b08ccac3d9ff3a24e0fa1ada9ec0e0a9f7e08cdf7da2d5171 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8) 2024-09-17T00:03:10.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 124922 9.84 MB/sec execute 174 sec latency 135.908 ms 2024-09-17T00:03:10.653 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:10 smithi194 podman[51367]: 2024-09-17 00:03:10.395628835 +0000 UTC m=+1.005263738 container cleanup 32c4fa431c9c102b08ccac3d9ff3a24e0fa1ada9ec0e0a9f7e08cdf7da2d5171 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , version=8, vcs-type=git, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, RELEASE=HEAD, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T00:03:10.656 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:10 smithi194 bash[51367]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194 2024-09-17T00:03:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:10 smithi194 podman[51380]: 2024-09-17 00:03:10.656047475 +0000 UTC m=+0.664157720 container remove 32c4fa431c9c102b08ccac3d9ff3a24e0fa1ada9ec0e0a9f7e08cdf7da2d5171 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , release=754, vcs-type=git, name=centos-stream, ceph=True, GIT_CLEAN=True, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, distribution-scope=public, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.display-name=CentOS Stream 8) 2024-09-17T00:03:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:10 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi194.service: Deactivated successfully. 2024-09-17T00:03:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:10 smithi194 systemd[1]: Stopped Ceph mon.smithi194 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:03:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:10 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi194.service: Consumed 7.397s CPU time. 2024-09-17T00:03:11.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 125189 9.79 MB/sec execute 175 sec latency 51.484 ms 2024-09-17T00:03:11.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 systemd[1]: Starting Ceph mon.smithi194 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:03:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 podman[51498]: 2024-09-17 00:03:11.140915402 +0000 UTC m=+0.017707900 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:03:11.885 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 podman[51498]: 2024-09-17 00:03:11.624467689 +0000 UTC m=+0.501260244 container create 3c19a28e308b78aebe102cc99f9fbc5e1451ec743d302ee30dfa7a13fec6f1b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-17T00:03:11.885 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 podman[51498]: 2024-09-17 00:03:11.860585685 +0000 UTC m=+0.737378171 container init 3c19a28e308b78aebe102cc99f9fbc5e1451ec743d302ee30dfa7a13fec6f1b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T00:03:11.886 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 podman[51498]: 2024-09-17 00:03:11.864360186 +0000 UTC m=+0.741152678 container start 3c19a28e308b78aebe102cc99f9fbc5e1451ec743d302ee30dfa7a13fec6f1b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: set uid:gid to 167:167 (ceph:ceph) 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable), process ceph-mon, pid 2 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: pidfile_write: ignore empty --pid-file 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: load: jerasure load: lrc 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: RocksDB version: 7.9.2 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Git sha 0 2024-09-17T00:03:12.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Compile date 2024-09-16 15:09:42 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: DB SUMMARY 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: DB Session ID: AA4Y4DH7QN8AQ9MFEHBH 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: CURRENT file: CURRENT 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: IDENTITY file: IDENTITY 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1091 Bytes 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi194/store.db dir, Total Num: 1, files: 000027.sst 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi194/store.db: 000025.log size: 0 ; 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.error_if_exists: 0 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.create_if_missing: 0 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.paranoid_checks: 1 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-17T00:03:12.162 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.env: 0x555c638e4940 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.fs: PosixFileSystem 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.info_log: 0x555c65cbd8c0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_file_opening_threads: 16 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.statistics: (nil) 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.use_fsync: 0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_log_file_size: 0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.keep_log_file_num: 1000 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.recycle_log_file_num: 0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.allow_fallocate: 1 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.allow_mmap_reads: 0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.allow_mmap_writes: 0 2024-09-17T00:03:12.163 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.use_direct_reads: 0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.create_missing_column_families: 0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.db_log_dir: 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.wal_dir: 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.advise_random_on_open: 1 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.db_write_buffer_size: 0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.write_buffer_manager: 0x555c65cccdc0 2024-09-17T00:03:12.164 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.rate_limiter: (nil) 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.wal_recovery_mode: 2 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.enable_thread_tracking: 0 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.enable_pipelined_write: 0 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.unordered_write: 0 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-17T00:03:12.165 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.row_cache: None 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.wal_filter: None 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.allow_ingest_behind: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.two_write_queues: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.manual_wal_flush: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.wal_compression: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.atomic_flush: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.log_readahead_size: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.best_efforts_recovery: 0 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-17T00:03:12.166 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.allow_data_in_errors: 0 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.db_host_id: __hostname__ 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_background_jobs: 2 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_background_compactions: -1 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_subcompactions: 1 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_total_wal_size: 0 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-17T00:03:12.167 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_open_files: -1 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bytes_per_sync: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_readahead_size: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_background_flushes: -1 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Compression algorithms supported: 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kZSTD supported: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kXpressCompression supported: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kBZip2Compression supported: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kLZ4Compression supported: 1 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kZlibCompression supported: 1 2024-09-17T00:03:12.168 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kLZ4HCCompression supported: 1 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: kSnappyCompression supported: 1 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi194/store.db/MANIFEST-000009 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.merge_operator: 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_filter: None 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_filter_factory: None 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.sst_partitioner_factory: None 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555c65cbd520) 2024-09-17T00:03:12.169 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: cache_index_and_filter_blocks: 1 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: pin_top_level_index_and_filter: 1 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_type: 0 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_block_index_type: 0 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_shortening: 1 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: checksum: 4 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: no_block_cache: 0 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache: 0x555c65cb11f0 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_name: BinnedLRUCache 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_options: 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: capacity : 536870912 2024-09-17T00:03:12.170 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: num_shard_bits : 4 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: strict_capacity_limit : 0 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: high_pri_pool_ratio: 0.000 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_cache_compressed: (nil) 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: persistent_cache: (nil) 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_size: 4096 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_size_deviation: 10 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_restart_interval: 16 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: index_block_restart_interval: 1 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: metadata_block_size: 4096 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: partition_filters: 0 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: use_delta_encoding: 1 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: filter_policy: bloomfilter 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: whole_key_filtering: 1 2024-09-17T00:03:12.171 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: verify_compression: 0 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: read_amp_bytes_per_bit: 0 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: format_version: 5 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: enable_index_compression: 1 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: block_align: 0 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_auto_readahead_size: 262144 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: prepopulate_block_cache: 0 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: initial_auto_readahead_size: 8192 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: num_file_reads_for_auto_readahead: 2 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.write_buffer_size: 33554432 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_write_buffer_number: 2 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression: NoCompression 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression: Disabled 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.prefix_extractor: nullptr 2024-09-17T00:03:12.172 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.num_levels: 7 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-17T00:03:12.173 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.level: 32767 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.strategy: 0 2024-09-17T00:03:12.174 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.enabled: false 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.target_file_size_base: 67108864 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-17T00:03:12.175 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.arena_block_size: 1048576 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.disable_auto_compactions: 0 2024-09-17T00:03:12.176 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.inplace_update_support: 0 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-17T00:03:12.177 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.bloom_locality: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.max_successive_merges: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.paranoid_file_checks: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.force_consistency_checks: 1 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.report_bg_io_stats: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.ttl: 2592000 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-17T00:03:12.178 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.enable_blob_files: false 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.min_blob_size: 0 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.blob_file_size: 268435456 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.blob_file_starting_level: 0 2024-09-17T00:03:12.179 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi194/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13989, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 02524b4f-e1ea-454d-b577-e3a2e61ad573 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726531391892592, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726531391892634, "job": 1, "event": "recovery_finished"} 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/version_set.cc:5047] Creating manifest 31 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-17T00:03:12.180 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555c65cdee00 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: DB pointer 0x555c65dda000 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** DB Stats ** 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-17T00:03:12.181 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: L6 1/0 9.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Sum 1/0 9.39 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** Compaction Stats [default] ** 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-17T00:03:12.182 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Block cache BinnedLRUCache@0x555c65cb11f0#2 capacity: 512.00 MB usage: 0.00 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Block cache entry stats(count,size,portion): Misc(1,0.00 KB,0%) 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: starting mon.smithi194 rank 1 at public addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] at bind addrs [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi194 fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???) e2 preinit fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).mds e7 new map 2024-09-17T00:03:12.183 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).mds e7 print_map 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: e7 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: legacy client fscid: 1 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Filesystem 'foofs' (1) 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: fs_name foofs 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: epoch 5 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: created 2024-09-16T23:57:36.932009+0000 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: modified 2024-09-16T23:57:43.146036+0000 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: tableserver 0 2024-09-17T00:03:12.184 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: root 0 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: session_timeout 60 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: session_autoclose 300 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_file_size 1099511627776 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_xattr_size 65536 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: required_client_features {} 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: last_failure 0 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: last_failure_osd_epoch 0 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: max_mds 1 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: in 0 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: up {0=14476} 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: failed 2024-09-17T00:03:12.185 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: damaged 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: stopped 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: data_pools [3] 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: metadata_pool 2 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: inline_data disabled 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: balancer 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: bal_rank_mask -1 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: standby_count_wanted 1 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: qdb_cluster leader: 0 members: 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: [mds.foofs.smithi007.fkkfbl{0:14476} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.7:6834/4061963992,v1:172.21.15.7:6835/4061963992] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: Standby daemons: 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: 2024-09-17T00:03:12.186 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout: [mds.foofs.smithi194.emredf{-1:24313} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] compat {c=[1],r=[1],i=[7ff]}] 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 ceph-mon[51517]: mon.smithi194@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 bash[51498]: 3c19a28e308b78aebe102cc99f9fbc5e1451ec743d302ee30dfa7a13fec6f1b8 2024-09-17T00:03:12.187 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:11 smithi194 systemd[1]: Started Ceph mon.smithi194 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:03:12.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 125459 9.77 MB/sec execute 176 sec latency 47.723 ms 2024-09-17T00:03:12.422 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:03:13.175 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-17T00:03:13.175 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:03:13.175 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T00:03:13.175 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: mgrmap e42: smithi007.czdphd(active, since 17s), standbys: smithi194.xlteft 2024-09-17T00:03:13.175 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: overall HEALTH_OK 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: mon.smithi007 calling monitor election 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: mon.smithi194 calling monitor election 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: mon.smithi007 is new leader, mons smithi007,smithi194 in quorum (ranks 0,1) 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: monmap epoch 3 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: last_changed 2024-09-17T00:03:12.028824+0000 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: created 2024-09-16T23:50:21.589767+0000 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: min_mon_release 19 (squid) 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: election_strategy: 1 2024-09-17T00:03:13.176 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: 0: [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] mon.smithi007 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: mgrmap e42: smithi007.czdphd(active, since 17s), standbys: smithi194.xlteft 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: overall HEALTH_OK 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: from='client.34126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:13.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:13 smithi007 ceph-mon[65487]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.6 MiB/s wr, 562 op/s 2024-09-17T00:03:13.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 125725 9.76 MB/sec execute 177 sec latency 45.515 ms 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: mgrmap e42: smithi007.czdphd(active, since 17s), standbys: smithi194.xlteft 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: overall HEALTH_OK 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: mon.smithi007 calling monitor election 2024-09-17T00:03:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: mon.smithi194 calling monitor election 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: mon.smithi007 is new leader, mons smithi007,smithi194 in quorum (ranks 0,1) 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi007"}]: dispatch 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mon metadata", "id": "smithi194"}]: dispatch 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: monmap epoch 3 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: fsid 5c15e83c-7486-11ef-bceb-c7b262605968 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: last_changed 2024-09-17T00:03:12.028824+0000 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: created 2024-09-16T23:50:21.589767+0000 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: min_mon_release 19 (squid) 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: election_strategy: 1 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: 0: [v2:172.21.15.7:3300/0,v1:172.21.15.7:6789/0] mon.smithi007 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: 1: [v2:172.21.15.194:3300/0,v1:172.21.15.194:6789/0] mon.smithi194 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:03:13.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: osdmap e72: 8 total, 8 up, 8 in 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: mgrmap e42: smithi007.czdphd(active, since 17s), standbys: smithi194.xlteft 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: overall HEALTH_OK 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: from='client.34126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:13.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:13 smithi194 ceph-mon[51517]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 26 KiB/s rd, 5.6 MiB/s wr, 562 op/s 2024-09-17T00:03:13.453 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:03:13.453 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (2m) 10s ago 11m 28.1M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:03:13.453 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (11m) 10s ago 11m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:03:13.453 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (10m) 14s ago 10m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:03:13.453 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (2m) 10s ago 11m 52.8M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:03:13.453 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (5m) 10s ago 5m 37.1M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (5m) 14s ago 5m 14.0M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (97s) 10s ago 12m 474M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (47s) 14s ago 10m 437M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (29s) 10s ago 12m 31.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 starting - - - 2048M 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (5m) 10s ago 5m 254M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (5m) 14s ago 5m 87.4M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (2m) 10s ago 11m 8120k - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (108s) 14s ago 10m 9365k - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (9m) 10s ago 9m 448M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (9m) 10s ago 9m 392M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (9m) 10s ago 9m 317M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:03:13.454 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (8m) 10s ago 8m 406M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:03:13.455 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (8m) 14s ago 8m 415M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:03:13.455 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (7m) 14s ago 7m 453M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:03:13.455 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (7m) 14s ago 7m 413M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:03:13.455 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (7m) 14s ago 7m 498M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:03:13.455 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (53s) 10s ago 10m 52.4M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:03:14.079 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:03:14.080 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:03:14.188 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:14 smithi007 ceph-mon[65487]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:14.188 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:14 smithi007 ceph-mon[65487]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:14.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 126007 9.73 MB/sec execute 178 sec latency 71.492 ms 2024-09-17T00:03:14.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:14 smithi194 ceph-mon[51517]: from='client.34130 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:14 smithi194 ceph-mon[51517]: from='client.34134 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:03:14.613 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "3/23 daemons upgraded", 2024-09-17T00:03:14.614 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T00:03:14.614 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:03:14.614 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:03:15.191 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:03:15.207 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:15 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3738852030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:15.207 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:15 smithi007 ceph-mon[65487]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 385 op/s 2024-09-17T00:03:15.208 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:15 smithi007 ceph-mon[65487]: from='client.34142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:15.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 126407 9.74 MB/sec execute 179 sec latency 53.873 ms 2024-09-17T00:03:15.361 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:15 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3738852030' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:15.361 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:15 smithi194 ceph-mon[51517]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.1 MiB/s wr, 385 op/s 2024-09-17T00:03:15.361 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:15 smithi194 ceph-mon[51517]: from='client.34142 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:16.235 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:16 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/444540592' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:03:16.235 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:16 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:16.235 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:16 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:16.281 INFO:teuthology.orchestra.run.smithi007.stdout: 5 126712 9.72 MB/sec execute 180 sec latency 61.223 ms 2024-09-17T00:03:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:16 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/444540592' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:03:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:16 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:16 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:17.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 127077 9.74 MB/sec execute 181 sec latency 94.045 ms 2024-09-17T00:03:17.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:17 smithi007 ceph-mon[65487]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 578 op/s 2024-09-17T00:03:17.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:17 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:17.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:17 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:17.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:17 smithi194 ceph-mon[51517]: pgmap v14: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 5.6 MiB/s wr, 578 op/s 2024-09-17T00:03:17.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:17 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:17.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:17 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:18.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 127408 9.75 MB/sec execute 182 sec latency 44.219 ms 2024-09-17T00:03:19.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 127823 9.79 MB/sec execute 183 sec latency 44.172 ms 2024-09-17T00:03:19.828 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:19 smithi007 ceph-mon[65487]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T00:03:20.021 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:19 smithi194 ceph-mon[51517]: pgmap v15: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T00:03:20.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 128079 9.77 MB/sec execute 184 sec latency 75.914 ms 2024-09-17T00:03:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:03:21.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: Reconfiguring mon.smithi007 (monmap changed)... 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: Reconfiguring daemon mon.smithi007 on smithi007 2024-09-17T00:03:21.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:21 smithi007 ceph-mon[65487]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T00:03:21.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 128397 9.75 MB/sec execute 185 sec latency 52.634 ms 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: Reconfiguring mon.smithi007 (monmap changed)... 2024-09-17T00:03:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:03:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:03:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: Reconfiguring daemon mon.smithi007 on smithi007 2024-09-17T00:03:21.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:21 smithi194 ceph-mon[51517]: pgmap v16: 97 pgs: 97 active+clean; 331 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.6 MiB/s wr, 370 op/s 2024-09-17T00:03:22.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 128650 9.73 MB/sec execute 186 sec latency 49.443 ms 2024-09-17T00:03:23.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 128899 9.70 MB/sec execute 187 sec latency 61.949 ms 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: pgmap v17: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 566 op/s 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: Reconfiguring mgr.smithi007.czdphd (monmap changed)... 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:23.708 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:23 smithi007 ceph-mon[65487]: Reconfiguring daemon mgr.smithi007.czdphd on smithi007 2024-09-17T00:03:23.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: pgmap v17: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.0 MiB/s wr, 566 op/s 2024-09-17T00:03:23.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:23.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:23.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: Reconfiguring mgr.smithi007.czdphd (monmap changed)... 2024-09-17T00:03:23.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi007.czdphd", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:03:23.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:03:23.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:23.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:23 smithi194 ceph-mon[51517]: Reconfiguring daemon mgr.smithi007.czdphd on smithi007 2024-09-17T00:03:24.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 129197 9.69 MB/sec execute 188 sec latency 46.307 ms 2024-09-17T00:03:25.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 129462 9.66 MB/sec execute 189 sec latency 73.154 ms 2024-09-17T00:03:25.761 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:25 smithi007 ceph-mon[65487]: pgmap v18: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 387 op/s 2024-09-17T00:03:25.761 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:03:25.762 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:25.762 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:25.762 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:03:25.762 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:25 smithi194 ceph-mon[51517]: pgmap v18: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.0 MiB/s wr, 387 op/s 2024-09-17T00:03:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:03:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:03:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:26.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 129910 9.66 MB/sec execute 190 sec latency 65.197 ms 2024-09-17T00:03:26.801 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:26 smithi007 ceph-mon[65487]: Reconfiguring crash.smithi007 (monmap changed)... 2024-09-17T00:03:26.801 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:26 smithi007 ceph-mon[65487]: Reconfiguring daemon crash.smithi007 on smithi007 2024-09-17T00:03:27.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:26 smithi194 ceph-mon[51517]: Reconfiguring crash.smithi007 (monmap changed)... 2024-09-17T00:03:27.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:26 smithi194 ceph-mon[51517]: Reconfiguring daemon crash.smithi007 on smithi007 2024-09-17T00:03:27.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 130176 9.64 MB/sec execute 191 sec latency 92.565 ms 2024-09-17T00:03:27.830 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:27 smithi007 ceph-mon[65487]: pgmap v19: 97 pgs: 97 active+clean; 325 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.4 MiB/s wr, 573 op/s 2024-09-17T00:03:27.830 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:27.830 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:27.830 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T00:03:27.830 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:27 smithi194 ceph-mon[51517]: pgmap v19: 97 pgs: 97 active+clean; 325 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.4 MiB/s wr, 573 op/s 2024-09-17T00:03:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T00:03:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:28.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 130524 9.66 MB/sec execute 192 sec latency 62.206 ms 2024-09-17T00:03:28.881 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:28 smithi007 ceph-mon[65487]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T00:03:28.881 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:28 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.0 on smithi007 2024-09-17T00:03:29.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:28 smithi194 ceph-mon[51517]: Reconfiguring osd.0 (monmap changed)... 2024-09-17T00:03:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:28 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.0 on smithi007 2024-09-17T00:03:29.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 130783 9.67 MB/sec execute 193 sec latency 57.872 ms 2024-09-17T00:03:29.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:29 smithi007 ceph-mon[65487]: pgmap v20: 97 pgs: 97 active+clean; 325 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 381 op/s 2024-09-17T00:03:29.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:29.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:29.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T00:03:29.905 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:30.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:29 smithi194 ceph-mon[51517]: pgmap v20: 97 pgs: 97 active+clean; 325 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.9 MiB/s wr, 381 op/s 2024-09-17T00:03:30.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:30.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:30.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T00:03:30.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:30.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 131193 9.70 MB/sec execute 194 sec latency 52.913 ms 2024-09-17T00:03:30.919 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:30 smithi007 ceph-mon[65487]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T00:03:30.919 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:30 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.1 on smithi007 2024-09-17T00:03:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:30 smithi194 ceph-mon[51517]: Reconfiguring osd.1 (monmap changed)... 2024-09-17T00:03:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:30 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.1 on smithi007 2024-09-17T00:03:31.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 131504 9.70 MB/sec execute 195 sec latency 47.000 ms 2024-09-17T00:03:31.931 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:31 smithi007 ceph-mon[65487]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 523 op/s 2024-09-17T00:03:31.931 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:31.931 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:31.931 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T00:03:31.932 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:32.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:31 smithi194 ceph-mon[51517]: pgmap v21: 97 pgs: 97 active+clean; 332 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 523 op/s 2024-09-17T00:03:32.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:32.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:32.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T00:03:32.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:32.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 131724 9.68 MB/sec execute 196 sec latency 71.973 ms 2024-09-17T00:03:32.938 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:32 smithi007 ceph-mon[65487]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T00:03:32.939 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:32 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.2 on smithi007 2024-09-17T00:03:33.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:32 smithi194 ceph-mon[51517]: Reconfiguring osd.2 (monmap changed)... 2024-09-17T00:03:33.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:32 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.2 on smithi007 2024-09-17T00:03:33.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 131972 9.64 MB/sec execute 197 sec latency 70.236 ms 2024-09-17T00:03:33.725 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:33 smithi007 ceph-mon[65487]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 570 op/s 2024-09-17T00:03:33.725 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:33 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:33.725 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:33 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:33.725 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:33 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T00:03:33.725 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:33 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:33 smithi194 ceph-mon[51517]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.1 MiB/s wr, 570 op/s 2024-09-17T00:03:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:33 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:33 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:33 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T00:03:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:33 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:34.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 132199 9.62 MB/sec execute 198 sec latency 51.787 ms 2024-09-17T00:03:34.748 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:34 smithi007 ceph-mon[65487]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T00:03:34.748 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:34 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.3 on smithi007 2024-09-17T00:03:35.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:34 smithi194 ceph-mon[51517]: Reconfiguring osd.3 (monmap changed)... 2024-09-17T00:03:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:34 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.3 on smithi007 2024-09-17T00:03:35.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 132435 9.59 MB/sec execute 199 sec latency 62.277 ms 2024-09-17T00:03:35.784 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:35 smithi007 ceph-mon[65487]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-09-17T00:03:36.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:35 smithi194 ceph-mon[51517]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 375 op/s 2024-09-17T00:03:36.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 132693 9.57 MB/sec execute 200 sec latency 56.954 ms 2024-09-17T00:03:37.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 132964 9.56 MB/sec execute 201 sec latency 106.664 ms 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: Reconfiguring mds.foofs.smithi007.fkkfbl (monmap changed)... 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: Reconfiguring daemon mds.foofs.smithi007.fkkfbl on smithi007 2024-09-17T00:03:37.342 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:37 smithi007 ceph-mon[65487]: pgmap v24: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: Reconfiguring mds.foofs.smithi007.fkkfbl (monmap changed)... 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: Reconfiguring daemon mds.foofs.smithi007.fkkfbl on smithi007 2024-09-17T00:03:37.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:37 smithi194 ceph-mon[51517]: pgmap v24: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.3 MiB/s wr, 544 op/s 2024-09-17T00:03:38.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 133398 9.55 MB/sec execute 202 sec latency 57.722 ms 2024-09-17T00:03:39.143 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:39.143 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:39.143 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-17T00:03:39.143 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:03:39.144 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:39.144 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-17T00:03:39.144 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:39 smithi007 ceph-mon[65487]: pgmap v25: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 358 op/s 2024-09-17T00:03:39.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 133734 9.55 MB/sec execute 203 sec latency 56.218 ms 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: Reconfiguring crash.smithi194 (monmap changed)... 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: Reconfiguring daemon crash.smithi194 on smithi194 2024-09-17T00:03:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:39 smithi194 ceph-mon[51517]: pgmap v25: 97 pgs: 97 active+clean; 326 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.9 MiB/s wr, 358 op/s 2024-09-17T00:03:40.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 134078 9.57 MB/sec execute 204 sec latency 55.884 ms 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: Reconfiguring mgr.smithi194.xlteft (monmap changed)... 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: Reconfiguring daemon mgr.smithi194.xlteft on smithi194 2024-09-17T00:03:40.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:03:41.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:41.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:41.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: Reconfiguring mgr.smithi194.xlteft (monmap changed)... 2024-09-17T00:03:41.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi194.xlteft", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-17T00:03:41.096 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-17T00:03:41.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:41.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: Reconfiguring daemon mgr.smithi194.xlteft on smithi194 2024-09-17T00:03:41.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:03:41.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 134311 9.57 MB/sec execute 205 sec latency 67.617 ms 2024-09-17T00:03:41.881 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:41 smithi194 ceph-mon[51517]: pgmap v26: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 408 op/s 2024-09-17T00:03:41.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:41 smithi007 ceph-mon[65487]: pgmap v26: 97 pgs: 97 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.8 MiB/s wr, 408 op/s 2024-09-17T00:03:42.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 134689 9.58 MB/sec execute 206 sec latency 91.173 ms 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-17T00:03:43.183 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:42 smithi007 ceph-mon[65487]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-17T00:03:43.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 134979 9.59 MB/sec execute 207 sec latency 41.786 ms 2024-09-17T00:03:43.406 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:43.406 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:43.406 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: Reconfiguring mon.smithi194 (monmap changed)... 2024-09-17T00:03:43.406 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-17T00:03:43.406 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-17T00:03:43.406 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:43.407 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: Reconfiguring daemon mon.smithi194 on smithi194 2024-09-17T00:03:43.407 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:42 smithi194 ceph-mon[51517]: pgmap v27: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 406 op/s 2024-09-17T00:03:44.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 135211 9.58 MB/sec execute 208 sec latency 58.676 ms 2024-09-17T00:03:44.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:44 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:44.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:44 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:44.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:44 smithi194 ceph-mon[51517]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T00:03:44.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:44 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T00:03:44.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:44 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:44.463 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:44 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.4 on smithi194 2024-09-17T00:03:44.705 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:44 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:44.705 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:44 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:44.705 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:44 smithi007 ceph-mon[65487]: Reconfiguring osd.4 (monmap changed)... 2024-09-17T00:03:44.705 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:44 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T00:03:44.705 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:44 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:44.705 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:44 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.4 on smithi194 2024-09-17T00:03:45.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 135507 9.55 MB/sec execute 209 sec latency 52.775 ms 2024-09-17T00:03:45.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:45 smithi007 ceph-mon[65487]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 359 op/s 2024-09-17T00:03:45.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:45.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:45.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T00:03:45.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:45.736 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:03:45.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:45 smithi194 ceph-mon[51517]: pgmap v28: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.3 MiB/s wr, 359 op/s 2024-09-17T00:03:45.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:45.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:45.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T00:03:45.785 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:46.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 135751 9.53 MB/sec execute 210 sec latency 59.537 ms 2024-09-17T00:03:46.505 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:46 smithi007 ceph-mon[65487]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T00:03:46.505 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:46 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.5 on smithi194 2024-09-17T00:03:46.505 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:46 smithi007 ceph-mon[65487]: from='client.34150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:46.798 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (3m) 43s ago 12m 28.1M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (11m) 43s ago 11m 6405k - 17.2.0 e1d6a67b021e 543b8b4e3bc9 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (11m) 30s ago 11m 6409k - 17.2.0 e1d6a67b021e ce62de3375b7 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (2m) 43s ago 11m 52.8M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (6m) 43s ago 6m 37.1M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (6m) 30s ago 6m 14.4M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:03:46.799 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (2m) 43s ago 13m 474M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (80s) 30s ago 11m 438M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (62s) 43s ago 13m 31.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (35s) 30s ago 11m 18.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (5m) 43s ago 5m 254M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (5m) 30s ago 5m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (2m) 43s ago 11m 8120k - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (2m) 30s ago 11m 16.9M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:03:46.800 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (10m) 43s ago 10m 448M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:03:46.801 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (10m) 43s ago 10m 392M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:03:46.801 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (9m) 43s ago 9m 317M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:03:46.801 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (9m) 43s ago 9m 406M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:03:46.801 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (8m) 30s ago 8m 434M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:03:46.801 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (8m) 30s ago 8m 476M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:03:46.801 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (7m) 30s ago 7m 424M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:03:46.802 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (7m) 30s ago 7m 532M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:03:46.802 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (86s) 43s ago 10m 52.4M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:03:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:46 smithi194 ceph-mon[51517]: Reconfiguring osd.5 (monmap changed)... 2024-09-17T00:03:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:46 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.5 on smithi194 2024-09-17T00:03:46.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:46 smithi194 ceph-mon[51517]: from='client.34150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:47.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 135990 9.50 MB/sec execute 211 sec latency 77.823 ms 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:47.517 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:03:47.518 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: pgmap v29: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:47.768 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:47 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/1677180297' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: from='client.44103 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: pgmap v29: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.4 MiB/s wr, 532 op/s 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:47.848 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:47 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/1677180297' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:03:48.049 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "4/23 daemons upgraded", 2024-09-17T00:03:48.050 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mon daemons", 2024-09-17T00:03:48.050 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:03:48.050 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:03:48.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 136364 9.51 MB/sec execute 212 sec latency 43.947 ms 2024-09-17T00:03:48.689 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.6 on smithi194 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: from='client.44107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:48.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:48 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2035677780' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:03:48.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: Reconfiguring osd.6 (monmap changed)... 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.6 on smithi194 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: from='client.44107 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:48.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:48 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2035677780' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:03:49.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 136850 9.49 MB/sec execute 213 sec latency 53.733 ms 2024-09-17T00:03:49.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:49 smithi007 ceph-mon[65487]: from='client.34166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:49.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:49 smithi007 ceph-mon[65487]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T00:03:49.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:49 smithi007 ceph-mon[65487]: Reconfiguring daemon osd.7 on smithi194 2024-09-17T00:03:49.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:49 smithi007 ceph-mon[65487]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 363 op/s 2024-09-17T00:03:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:49 smithi194 ceph-mon[51517]: from='client.34166 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:03:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:49 smithi194 ceph-mon[51517]: Reconfiguring osd.7 (monmap changed)... 2024-09-17T00:03:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:49 smithi194 ceph-mon[51517]: Reconfiguring daemon osd.7 on smithi194 2024-09-17T00:03:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:49 smithi194 ceph-mon[51517]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.8 MiB/s wr, 363 op/s 2024-09-17T00:03:50.282 INFO:teuthology.orchestra.run.smithi007.stdout: 5 137260 9.50 MB/sec execute 214 sec latency 60.776 ms 2024-09-17T00:03:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: Reconfiguring mds.foofs.smithi194.emredf (monmap changed)... 2024-09-17T00:03:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:03:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: Reconfiguring daemon mds.foofs.smithi194.emredf on smithi194 2024-09-17T00:03:51.065 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:50 smithi007 ceph-mon[65487]: pgmap v31: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 565 op/s 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: Reconfiguring mds.foofs.smithi194.emredf (monmap changed)... 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: Reconfiguring daemon mds.foofs.smithi194.emredf on smithi194 2024-09-17T00:03:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:50 smithi194 ceph-mon[51517]: pgmap v31: 97 pgs: 97 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 565 op/s 2024-09-17T00:03:51.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 137708 9.55 MB/sec execute 215 sec latency 50.505 ms 2024-09-17T00:03:52.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 137949 9.53 MB/sec execute 216 sec latency 64.695 ms 2024-09-17T00:03:52.353 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:52.353 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:52.353 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:03:52.353 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:52.353 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:52.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all mon 2024-09-17T00:03:52.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:52.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi007"}]: dispatch 2024-09-17T00:03:52.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi007"}]': finished 2024-09-17T00:03:52.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]: dispatch 2024-09-17T00:03:52.354 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:52 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]': finished 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all mon 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi007"}]: dispatch 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi007"}]': finished 2024-09-17T00:03:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]: dispatch 2024-09-17T00:03:52.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:52 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi194"}]': finished 2024-09-17T00:03:53.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 138330 9.56 MB/sec execute 217 sec latency 85.311 ms 2024-09-17T00:03:53.362 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:53 smithi007 ceph-mon[65487]: pgmap v32: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 543 op/s 2024-09-17T00:03:53.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:53 smithi194 ceph-mon[51517]: pgmap v32: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.8 MiB/s wr, 543 op/s 2024-09-17T00:03:54.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 138684 9.56 MB/sec execute 218 sec latency 58.184 ms 2024-09-17T00:03:54.644 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:54 smithi007 ceph-mon[65487]: Upgrade: Updating crash.smithi007 (1/2) 2024-09-17T00:03:54.644 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:54.644 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:03:54.644 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:54.644 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:54 smithi007 ceph-mon[65487]: Deploying daemon crash.smithi007 on smithi007 2024-09-17T00:03:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:54 smithi194 ceph-mon[51517]: Upgrade: Updating crash.smithi007 (1/2) 2024-09-17T00:03:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:03:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi007", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:03:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:03:54.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:54 smithi194 ceph-mon[51517]: Deploying daemon crash.smithi007 on smithi007 2024-09-17T00:03:55.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 139003 9.55 MB/sec execute 219 sec latency 86.560 ms 2024-09-17T00:03:55.656 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:55 smithi007 ceph-mon[65487]: pgmap v33: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 402 op/s 2024-09-17T00:03:55.656 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:03:55.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:55 smithi194 ceph-mon[51517]: pgmap v33: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 402 op/s 2024-09-17T00:03:55.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:03:56.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 139330 9.53 MB/sec execute 220 sec latency 48.076 ms 2024-09-17T00:03:57.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 139653 9.52 MB/sec execute 221 sec latency 53.259 ms 2024-09-17T00:03:57.967 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:57 smithi007 ceph-mon[65487]: pgmap v34: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-09-17T00:03:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:57 smithi194 ceph-mon[51517]: pgmap v34: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-09-17T00:03:58.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 140011 9.51 MB/sec execute 222 sec latency 60.399 ms 2024-09-17T00:03:59.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 140330 9.49 MB/sec execute 223 sec latency 64.268 ms 2024-09-17T00:03:59.727 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:03:59 smithi007 ceph-mon[65487]: pgmap v35: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-17T00:04:00.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:03:59 smithi194 ceph-mon[51517]: pgmap v35: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.4 MiB/s wr, 433 op/s 2024-09-17T00:04:00.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 140662 9.49 MB/sec execute 224 sec latency 65.868 ms 2024-09-17T00:04:01.255 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:01 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:01.255 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:01 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:01.255 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:01 smithi007 ceph-mon[65487]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.3 MiB/s wr, 526 op/s 2024-09-17T00:04:01.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 141025 9.50 MB/sec execute 225 sec latency 41.893 ms 2024-09-17T00:04:01.399 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:01 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:01.399 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:01 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:01.399 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:01 smithi194 ceph-mon[51517]: pgmap v36: 97 pgs: 97 active+clean; 329 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.3 MiB/s wr, 526 op/s 2024-09-17T00:04:02.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 141395 9.52 MB/sec execute 226 sec latency 51.784 ms 2024-09-17T00:04:03.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:02 smithi007 ceph-mon[65487]: Upgrade: Updating crash.smithi194 (2/2) 2024-09-17T00:04:03.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:02 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:03.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:02 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:04:03.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:02 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:03.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:02 smithi007 ceph-mon[65487]: Deploying daemon crash.smithi194 on smithi194 2024-09-17T00:04:03.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:02 smithi007 ceph-mon[65487]: pgmap v37: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-17T00:04:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:02 smithi194 ceph-mon[51517]: Upgrade: Updating crash.smithi194 (2/2) 2024-09-17T00:04:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:02 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:02 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi194", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-17T00:04:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:02 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:02 smithi194 ceph-mon[51517]: Deploying daemon crash.smithi194 on smithi194 2024-09-17T00:04:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:02 smithi194 ceph-mon[51517]: pgmap v37: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.6 MiB/s wr, 445 op/s 2024-09-17T00:04:03.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 141718 9.53 MB/sec execute 227 sec latency 62.117 ms 2024-09-17T00:04:04.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 142005 9.54 MB/sec execute 228 sec latency 57.512 ms 2024-09-17T00:04:05.283 INFO:teuthology.orchestra.run.smithi007.stdout: 5 142346 9.53 MB/sec execute 229 sec latency 54.937 ms 2024-09-17T00:04:05.795 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:05 smithi194 ceph-mon[51517]: pgmap v38: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-17T00:04:05.887 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:05 smithi007 ceph-mon[65487]: pgmap v38: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 416 op/s 2024-09-17T00:04:06.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 142640 9.51 MB/sec execute 230 sec latency 54.177 ms 2024-09-17T00:04:06.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:06.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:06.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:06 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:06.899 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:06 smithi007 ceph-mon[65487]: pgmap v39: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-09-17T00:04:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:06 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:06 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:06 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:06 smithi194 ceph-mon[51517]: pgmap v39: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.5 MiB/s wr, 607 op/s 2024-09-17T00:04:07.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 142920 9.50 MB/sec execute 231 sec latency 53.713 ms 2024-09-17T00:04:08.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 143282 9.49 MB/sec execute 232 sec latency 42.784 ms 2024-09-17T00:04:09.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 143634 9.48 MB/sec execute 233 sec latency 95.550 ms 2024-09-17T00:04:09.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:09 smithi194 ceph-mon[51517]: pgmap v40: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-17T00:04:09.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:09.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:09 smithi007 ceph-mon[65487]: pgmap v40: 97 pgs: 97 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 403 op/s 2024-09-17T00:04:09.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:10.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 144060 9.47 MB/sec execute 234 sec latency 47.961 ms 2024-09-17T00:04:11.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 144282 9.46 MB/sec execute 235 sec latency 113.516 ms 2024-09-17T00:04:11.303 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.303 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.303 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:11 smithi007 ceph-mon[65487]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.4 MiB/s wr, 592 op/s 2024-09-17T00:04:11.303 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.303 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:11 smithi194 ceph-mon[51517]: pgmap v41: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 5.4 MiB/s wr, 592 op/s 2024-09-17T00:04:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:12.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 144654 9.47 MB/sec execute 236 sec latency 50.516 ms 2024-09-17T00:04:13.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 145007 9.49 MB/sec execute 237 sec latency 43.261 ms 2024-09-17T00:04:13.846 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:13 smithi007 ceph-mon[65487]: pgmap v42: 97 pgs: 97 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 526 op/s 2024-09-17T00:04:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:13 smithi194 ceph-mon[51517]: pgmap v42: 97 pgs: 97 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.9 MiB/s wr, 526 op/s 2024-09-17T00:04:14.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 145275 9.50 MB/sec execute 238 sec latency 97.595 ms 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: pgmap v43: 97 pgs: 97 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 407 op/s 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.254 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 145497 9.50 MB/sec execute 239 sec latency 61.287 ms 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: pgmap v43: 97 pgs: 97 active+clean; 325 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 407 op/s 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:16.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 145826 9.48 MB/sec execute 240 sec latency 52.185 ms 2024-09-17T00:04:17.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 146184 9.47 MB/sec execute 241 sec latency 53.885 ms 2024-09-17T00:04:17.812 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:17 smithi007 ceph-mon[65487]: pgmap v44: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.2 MiB/s wr, 576 op/s 2024-09-17T00:04:17.958 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:17 smithi194 ceph-mon[51517]: pgmap v44: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.2 MiB/s wr, 576 op/s 2024-09-17T00:04:18.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 146458 9.46 MB/sec execute 242 sec latency 55.927 ms 2024-09-17T00:04:19.252 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:04:19.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 146819 9.46 MB/sec execute 243 sec latency 53.403 ms 2024-09-17T00:04:19.863 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:19 smithi007 ceph-mon[65487]: pgmap v45: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 385 op/s 2024-09-17T00:04:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:19 smithi194 ceph-mon[51517]: pgmap v45: 97 pgs: 97 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.9 MiB/s wr, 385 op/s 2024-09-17T00:04:20.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 147203 9.44 MB/sec execute 244 sec latency 45.372 ms 2024-09-17T00:04:20.413 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:04:20.413 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (3m) 6s ago 12m 27.0M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:04:20.413 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (21s) 6s ago 12m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (15s) 10s ago 12m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (3m) 6s ago 12m 54.1M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (6m) 6s ago 6m 37.3M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (6m) 10s ago 6m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (2m) 6s ago 13m 493M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (114s) 10s ago 11m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (96s) 6s ago 13m 44.3M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (69s) 10s ago 11m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (6m) 6s ago 6m 251M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (6m) 10s ago 6m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (3m) 6s ago 12m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (2m) 10s ago 11m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:04:20.414 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (11m) 6s ago 11m 500M 1937M 17.2.0 e1d6a67b021e 78961710495e 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (10m) 6s ago 10m 435M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (10m) 6s ago 10m 364M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (9m) 6s ago 9m 460M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (9m) 10s ago 9m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (8m) 10s ago 8m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (8m) 10s ago 8m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (8m) 10s ago 8m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:04:20.415 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (119s) 6s ago 11m 59.9M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:04:20.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:20 smithi007 ceph-mon[65487]: from='client.34174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:21.070 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:04:21.071 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:04:21.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:20 smithi194 ceph-mon[51517]: from='client.34174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:21.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 147546 9.43 MB/sec execute 245 sec latency 42.865 ms 2024-09-17T00:04:21.614 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:04:21.614 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:04:21.614 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:04:21.614 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:04:21.614 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:04:21.614 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "6/23 daemons upgraded", 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:04:21.615 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.3 MiB/s wr, 579 op/s 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2359341848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:21.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi007"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi007"}]': finished 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]: dispatch 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]': finished 2024-09-17T00:04:21.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T00:04:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='client.34178 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='client.34182 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: pgmap v46: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.3 MiB/s wr, 579 op/s 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2359341848' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:22.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:22.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:22.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi007"}]: dispatch 2024-09-17T00:04:22.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi007"}]': finished 2024-09-17T00:04:22.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]: dispatch 2024-09-17T00:04:22.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi194"}]': finished 2024-09-17T00:04:22.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T00:04:22.205 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:04:22.284 INFO:teuthology.orchestra.run.smithi007.stdout: 5 147822 9.43 MB/sec execute 246 sec latency 41.321 ms 2024-09-17T00:04:22.964 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:22 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all crash 2024-09-17T00:04:22.964 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:22 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T00:04:22.964 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:22 smithi007 ceph-mon[65487]: Upgrade: osd.0 is safe to restart 2024-09-17T00:04:22.964 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:22 smithi007 ceph-mon[65487]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:22.964 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:22 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3135313352' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:04:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:22 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all crash 2024-09-17T00:04:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:22 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-17T00:04:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:22 smithi194 ceph-mon[51517]: Upgrade: osd.0 is safe to restart 2024-09-17T00:04:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:22 smithi194 ceph-mon[51517]: from='client.44135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:22 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3135313352' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:04:23.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 148052 9.43 MB/sec execute 247 sec latency 62.243 ms 2024-09-17T00:04:23.997 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:23 smithi007 ceph-mon[65487]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 417 op/s 2024-09-17T00:04:24.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:23 smithi194 ceph-mon[51517]: pgmap v47: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 417 op/s 2024-09-17T00:04:24.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 148374 9.42 MB/sec execute 248 sec latency 46.737 ms 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: Upgrade: Updating osd.0 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: Deploying daemon osd.0 on smithi007 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-09-17T00:04:25.073 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: Upgrade: Updating osd.0 2024-09-17T00:04:25.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:25.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-17T00:04:25.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:25.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: Deploying daemon osd.0 on smithi007 2024-09-17T00:04:25.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: pgmap v48: 97 pgs: 97 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-09-17T00:04:25.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:25.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 148635 9.43 MB/sec execute 249 sec latency 66.995 ms 2024-09-17T00:04:26.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 148832 9.43 MB/sec execute 250 sec latency 89.725 ms 2024-09-17T00:04:27.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 149064 9.43 MB/sec execute 251 sec latency 88.488 ms 2024-09-17T00:04:27.897 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:27 smithi007 ceph-mon[65487]: pgmap v49: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 552 op/s 2024-09-17T00:04:27.897 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:27 smithi007 ceph-mon[65487]: mgrmap e43: smithi007.czdphd(active, since 92s), standbys: smithi194.xlteft 2024-09-17T00:04:27.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:27 smithi194 ceph-mon[51517]: pgmap v49: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.9 MiB/s wr, 552 op/s 2024-09-17T00:04:27.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:27 smithi194 ceph-mon[51517]: mgrmap e43: smithi007.czdphd(active, since 92s), standbys: smithi194.xlteft 2024-09-17T00:04:28.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 149262 9.40 MB/sec execute 252 sec latency 79.365 ms 2024-09-17T00:04:29.179 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:29 smithi007 systemd[1]: Stopping Ceph osd.0 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:04:29.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 149489 9.39 MB/sec execute 253 sec latency 75.790 ms 2024-09-17T00:04:29.432 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:29 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[34309]: 2024-09-17T00:04:29.389+0000 7f9927a19700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:04:29.432 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:29 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[34309]: 2024-09-17T00:04:29.389+0000 7f9927a19700 -1 osd.0 72 *** Got signal Terminated *** 2024-09-17T00:04:29.432 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:29 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[34309]: 2024-09-17T00:04:29.389+0000 7f9927a19700 -1 osd.0 72 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:04:30.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:29 smithi007 ceph-mon[65487]: pgmap v50: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 382 op/s 2024-09-17T00:04:30.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:29 smithi007 ceph-mon[65487]: osd.0 marked itself down and dead 2024-09-17T00:04:30.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:29 smithi194 ceph-mon[51517]: pgmap v50: 97 pgs: 97 active+clean; 329 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.7 MiB/s wr, 382 op/s 2024-09-17T00:04:30.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:29 smithi194 ceph-mon[51517]: osd.0 marked itself down and dead 2024-09-17T00:04:30.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 149596 9.35 MB/sec execute 254 sec latency 585.494 ms 2024-09-17T00:04:30.636 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:30 smithi007 podman[70975]: 2024-09-17 00:04:30.366316042 +0000 UTC m=+1.105505962 container died 78961710495eb826fe04f3daa136814ab553687d8cbad574ecdffafa595cde26 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, name=centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.tags=base centos centos-stream, distribution-scope=public, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, io.buildah.version=1.19.8, version=8, build-date=2022-05-03T08:36:31.336870) 2024-09-17T00:04:30.891 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:30 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:04:30.891 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:30 smithi007 ceph-mon[65487]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T00:04:30.891 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:30 smithi007 ceph-mon[65487]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T00:04:30.891 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:30 smithi007 podman[70975]: 2024-09-17 00:04:30.746153556 +0000 UTC m=+1.485343476 container cleanup 78961710495eb826fe04f3daa136814ab553687d8cbad574ecdffafa595cde26 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vcs-type=git, ceph=True, architecture=x86_64, release=754, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream) 2024-09-17T00:04:30.891 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:30 smithi007 bash[70975]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0 2024-09-17T00:04:31.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:30 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:04:31.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:30 smithi194 ceph-mon[51517]: osdmap e73: 8 total, 7 up, 8 in 2024-09-17T00:04:31.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:30 smithi194 ceph-mon[51517]: osdmap e74: 8 total, 7 up, 8 in 2024-09-17T00:04:31.151 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:31 smithi007 podman[70991]: 2024-09-17 00:04:31.108896072 +0000 UTC m=+0.736088694 container remove 78961710495eb826fe04f3daa136814ab553687d8cbad574ecdffafa595cde26 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., maintainer=Guillaume Abrioux , io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_BRANCH=HEAD, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, architecture=x86_64, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.component=centos-stream-container, RELEASE=HEAD, name=centos-stream) 2024-09-17T00:04:31.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 149756 9.33 MB/sec execute 255 sec latency 1047.210 ms 2024-09-17T00:04:31.655 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:31 smithi007 podman[71080]: 2024-09-17 00:04:31.431679421 +0000 UTC m=+0.022124750 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:04:31.655 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:31 smithi007 podman[71080]: 2024-09-17 00:04:31.549093225 +0000 UTC m=+0.139538541 container create aa578e3025d063eb67ad0f9a9febaff6e995132e3440e8d4de9bf87e474b30ca (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2) 2024-09-17T00:04:31.912 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:31 smithi007 podman[71080]: 2024-09-17 00:04:31.885180566 +0000 UTC m=+0.475625889 container init aa578e3025d063eb67ad0f9a9febaff6e995132e3440e8d4de9bf87e474b30ca (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:04:31.913 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:31 smithi007 podman[71080]: 2024-09-17 00:04:31.89718569 +0000 UTC m=+0.487631008 container start aa578e3025d063eb67ad0f9a9febaff6e995132e3440e8d4de9bf87e474b30ca (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T00:04:31.914 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:31 smithi007 ceph-mon[65487]: pgmap v52: 97 pgs: 25 peering, 6 stale+active+clean, 66 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 337 op/s 2024-09-17T00:04:31.914 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:31 smithi007 ceph-mon[65487]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-17T00:04:32.070 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:31 smithi194 ceph-mon[51517]: pgmap v52: 97 pgs: 25 peering, 6 stale+active+clean, 66 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 3.8 MiB/s wr, 337 op/s 2024-09-17T00:04:32.070 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:31 smithi194 ceph-mon[51517]: Health check failed: Reduced data availability: 3 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-17T00:04:32.164 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:32 smithi007 podman[71080]: 2024-09-17 00:04:32.011175393 +0000 UTC m=+0.601620716 container attach aa578e3025d063eb67ad0f9a9febaff6e995132e3440e8d4de9bf87e474b30ca (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T00:04:32.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 150094 9.33 MB/sec execute 256 sec latency 70.072 ms 2024-09-17T00:04:32.416 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:32 smithi007 podman[71080]: 2024-09-17 00:04:32.232001022 +0000 UTC m=+0.822446345 container died aa578e3025d063eb67ad0f9a9febaff6e995132e3440e8d4de9bf87e474b30ca (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-17T00:04:33.174 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 podman[71102]: 2024-09-17 00:04:33.067614965 +0000 UTC m=+0.828794764 container remove aa578e3025d063eb67ad0f9a9febaff6e995132e3440e8d4de9bf87e474b30ca (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T00:04:33.175 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-17T00:04:33.175 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 systemd[1]: Stopped Ceph osd.0 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:04:33.175 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 31.873s CPU time. 2024-09-17T00:04:33.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 150338 9.31 MB/sec execute 257 sec latency 50.555 ms 2024-09-17T00:04:33.679 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 systemd[1]: Starting Ceph osd.0 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:04:33.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:33 smithi007 ceph-mon[65487]: pgmap v54: 97 pgs: 36 peering, 61 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.8 MiB/s wr, 409 op/s 2024-09-17T00:04:33.930 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 podman[71203]: 2024-09-17 00:04:33.618889134 +0000 UTC m=+0.022164866 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:04:33.930 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:33 smithi007 podman[71203]: 2024-09-17 00:04:33.795030452 +0000 UTC m=+0.198306182 container create b52a2e50b1c501db0acbfb21af735f6987081e30df238e127c0623dbea9be9f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:04:34.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:33 smithi194 ceph-mon[51517]: pgmap v54: 97 pgs: 36 peering, 61 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.8 MiB/s wr, 409 op/s 2024-09-17T00:04:34.183 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 podman[71203]: 2024-09-17 00:04:34.15730525 +0000 UTC m=+0.560581005 container init b52a2e50b1c501db0acbfb21af735f6987081e30df238e127c0623dbea9be9f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T00:04:34.183 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 podman[71203]: 2024-09-17 00:04:34.171347483 +0000 UTC m=+0.574623219 container start b52a2e50b1c501db0acbfb21af735f6987081e30df238e127c0623dbea9be9f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-17T00:04:34.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 150885 9.31 MB/sec execute 258 sec latency 53.100 ms 2024-09-17T00:04:34.437 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 podman[71203]: 2024-09-17 00:04:34.316123323 +0000 UTC m=+0.719399064 container attach b52a2e50b1c501db0acbfb21af735f6987081e30df238e127c0623dbea9be9f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97) 2024-09-17T00:04:34.437 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:04:34.437 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 bash[71203]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:04:34.437 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:04:34.437 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:34 smithi007 bash[71203]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:04:35.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 151469 9.34 MB/sec execute 259 sec latency 54.519 ms 2024-09-17T00:04:35.695 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:35 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T00:04:35.695 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:35 smithi007 bash[71203]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T00:04:35.695 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:35 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T00:04:35.695 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:35 smithi007 bash[71203]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T00:04:35.949 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:35 smithi007 ceph-mon[65487]: pgmap v55: 97 pgs: 36 peering, 61 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.9 MiB/s wr, 165 op/s 2024-09-17T00:04:36.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:35 smithi194 ceph-mon[51517]: pgmap v55: 97 pgs: 36 peering, 61 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.9 MiB/s wr, 165 op/s 2024-09-17T00:04:36.200 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T00:04:36.200 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 bash[71203]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T00:04:36.200 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T00:04:36.200 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 bash[71203]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T00:04:36.200 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T00:04:36.201 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 bash[71203]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-17T00:04:36.201 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T00:04:36.201 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 bash[71203]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-17T00:04:36.201 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate[71216]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T00:04:36.201 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 bash[71203]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-17T00:04:36.201 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:36 smithi007 podman[71203]: 2024-09-17 00:04:36.100082416 +0000 UTC m=+2.503358164 container died b52a2e50b1c501db0acbfb21af735f6987081e30df238e127c0623dbea9be9f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0) 2024-09-17T00:04:36.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 152055 9.37 MB/sec execute 260 sec latency 82.614 ms 2024-09-17T00:04:36.957 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-mon[65487]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 555 op/s; 209/1644 objects degraded (12.713%) 2024-09-17T00:04:36.957 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 209/1644 objects degraded (12.713%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:36.957 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:36 smithi007 ceph-mon[65487]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-09-17T00:04:37.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:36 smithi194 ceph-mon[51517]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.3 MiB/s wr, 555 op/s; 209/1644 objects degraded (12.713%) 2024-09-17T00:04:37.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:36 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 209/1644 objects degraded (12.713%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:37.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:36 smithi194 ceph-mon[51517]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 12 pgs peering) 2024-09-17T00:04:37.213 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 podman[71367]: 2024-09-17 00:04:37.157237866 +0000 UTC m=+1.049242927 container remove b52a2e50b1c501db0acbfb21af735f6987081e30df238e127c0623dbea9be9f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:04:37.288 INFO:teuthology.orchestra.run.smithi007.stdout: 5 152445 9.40 MB/sec execute 261 sec latency 63.757 ms 2024-09-17T00:04:37.476 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 podman[71420]: 2024-09-17 00:04:37.299139657 +0000 UTC m=+0.019425709 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:04:37.476 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 podman[71420]: 2024-09-17 00:04:37.441846788 +0000 UTC m=+0.162132818 container create 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-17T00:04:37.998 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 podman[71420]: 2024-09-17 00:04:37.809209981 +0000 UTC m=+0.529496019 container init 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T00:04:37.998 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 podman[71420]: 2024-09-17 00:04:37.819224456 +0000 UTC m=+0.539510487 container start 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-17T00:04:37.998 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 ceph-osd[71439]: -- 172.21.15.7:0/2632482859 <== mon.0 v2:172.21.15.7:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ca5e82c1e0 con 0x55ca5da19c00 2024-09-17T00:04:37.999 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 bash[71420]: 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 2024-09-17T00:04:37.999 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:37 smithi007 systemd[1]: Started Ceph osd.0 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:04:38.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 152709 9.40 MB/sec execute 262 sec latency 60.100 ms 2024-09-17T00:04:39.257 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:39.258 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:39.258 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:39.258 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:39 smithi007 ceph-mon[65487]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 497 op/s; 209/1644 objects degraded (12.713%) 2024-09-17T00:04:39.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 152976 9.38 MB/sec execute 263 sec latency 103.864 ms 2024-09-17T00:04:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:39.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:39 smithi194 ceph-mon[51517]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 333 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.8 MiB/s wr, 497 op/s; 209/1644 objects degraded (12.713%) 2024-09-17T00:04:39.511 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:39 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:04:39.376+0000 7fbf64b09640 -1 Falling back to public interface 2024-09-17T00:04:40.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 153422 9.38 MB/sec execute 264 sec latency 63.674 ms 2024-09-17T00:04:40.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:40.776 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:41.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:41.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:41.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 153820 9.38 MB/sec execute 265 sec latency 78.731 ms 2024-09-17T00:04:41.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:41 smithi007 ceph-mon[65487]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 590 op/s; 222/1659 objects degraded (13.382%) 2024-09-17T00:04:42.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:41 smithi194 ceph-mon[51517]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 335 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 590 op/s; 222/1659 objects degraded (13.382%) 2024-09-17T00:04:42.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 154339 9.38 MB/sec execute 266 sec latency 38.578 ms 2024-09-17T00:04:42.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:42 smithi007 ceph-mon[65487]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 512 op/s; 220/1641 objects degraded (13.406%) 2024-09-17T00:04:42.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:42 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 220/1641 objects degraded (13.406%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:43.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:42 smithi194 ceph-mon[51517]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 5.3 MiB/s wr, 512 op/s; 220/1641 objects degraded (13.406%) 2024-09-17T00:04:43.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:42 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 220/1641 objects degraded (13.406%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:43.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 154598 9.36 MB/sec execute 267 sec latency 56.032 ms 2024-09-17T00:04:44.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 154876 9.37 MB/sec execute 268 sec latency 55.428 ms 2024-09-17T00:04:45.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 155261 9.38 MB/sec execute 269 sec latency 49.508 ms 2024-09-17T00:04:45.380 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:45 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:04:45.141+0000 7fbf64b09640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-17T00:04:45.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:45 smithi007 ceph-mon[65487]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 495 op/s; 220/1641 objects degraded (13.406%) 2024-09-17T00:04:46.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:45 smithi194 ceph-mon[51517]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 336 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.0 MiB/s wr, 495 op/s; 220/1641 objects degraded (13.406%) 2024-09-17T00:04:46.188 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:46 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:04:46.105+0000 7fbf64b09640 -1 osd.0 72 log_to_monitors true 2024-09-17T00:04:46.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 155536 9.38 MB/sec execute 270 sec latency 75.038 ms 2024-09-17T00:04:46.962 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:46 smithi007 ceph-mon[65487]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T00:04:47.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:46 smithi194 ceph-mon[51517]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-17T00:04:47.220 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:04:47.069+0000 7fbf588ac640 -1 osd.0 72 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:04:47.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 155788 9.38 MB/sec execute 271 sec latency 57.778 ms 2024-09-17T00:04:47.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 709 op/s; 215/1578 objects degraded (13.625%) 2024-09-17T00:04:47.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T00:04:47.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T00:04:47.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:04:47.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' 2024-09-17T00:04:47.736 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:47.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.9 MiB/s wr, 709 op/s; 215/1578 objects degraded (13.625%) 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: osdmap e75: 8 total, 7 up, 8 in 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: from='osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985]' entity='osd.0' 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:48.285 INFO:teuthology.orchestra.run.smithi007.stdout: 5 156047 9.38 MB/sec execute 272 sec latency 282.055 ms 2024-09-17T00:04:49.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 156102 9.35 MB/sec execute 273 sec latency 1282.086 ms 2024-09-17T00:04:49.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:49.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:04:49.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985] boot 2024-09-17T00:04:49.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T00:04:49.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:04:49.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:49.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:49 smithi007 ceph-mon[65487]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 355 op/s; 215/1578 objects degraded (13.625%) 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: osd.0 [v2:172.21.15.7:6802/2821718985,v1:172.21.15.7:6803/2821718985] boot 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: osdmap e76: 8 total, 8 up, 8 in 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:49.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:49 smithi194 ceph-mon[51517]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.0 MiB/s wr, 355 op/s; 215/1578 objects degraded (13.625%) 2024-09-17T00:04:50.215 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:50 smithi007 ceph-mon[65487]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T00:04:50.215 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:50 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 215/1578 objects degraded (13.625%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:50.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 156153 9.33 MB/sec execute 274 sec latency 2134.765 ms 2024-09-17T00:04:50.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:50 smithi194 ceph-mon[51517]: osdmap e77: 8 total, 8 up, 8 in 2024-09-17T00:04:50.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:50 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 215/1578 objects degraded (13.625%), 29 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:51.226 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:51 smithi007 ceph-mon[65487]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T00:04:51.226 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:51 smithi007 ceph-mon[65487]: pgmap v67: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 9 active+undersized+degraded, 73 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 252 op/s; 212/1575 objects degraded (13.460%); 72 KiB/s, 12 objects/s recovering 2024-09-17T00:04:51.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 156336 9.31 MB/sec execute 275 sec latency 61.054 ms 2024-09-17T00:04:51.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:51 smithi194 ceph-mon[51517]: osdmap e78: 8 total, 8 up, 8 in 2024-09-17T00:04:51.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:51 smithi194 ceph-mon[51517]: pgmap v67: 97 pgs: 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+undersized, 9 active+undersized+degraded, 73 active+clean; 332 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.2 MiB/s wr, 252 op/s; 212/1575 objects degraded (13.460%); 72 KiB/s, 12 objects/s recovering 2024-09-17T00:04:52.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 156562 9.29 MB/sec execute 276 sec latency 87.115 ms 2024-09-17T00:04:52.401 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:52 smithi007 ceph-mon[65487]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T00:04:52.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:52 smithi194 ceph-mon[51517]: osdmap e79: 8 total, 8 up, 8 in 2024-09-17T00:04:52.718 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:04:53.242 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:53 smithi007 ceph-mon[65487]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T00:04:53.242 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:53 smithi007 ceph-mon[65487]: pgmap v70: 97 pgs: 1 peering, 11 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 309 op/s; 181/1563 objects degraded (11.580%); 72 KiB/s, 23 objects/s recovering 2024-09-17T00:04:53.242 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:53 smithi007 ceph-mon[65487]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:53.243 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:53 smithi007 ceph-mon[65487]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T00:04:53.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 156871 9.29 MB/sec execute 277 sec latency 93.241 ms 2024-09-17T00:04:53.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:53 smithi194 ceph-mon[51517]: osdmap e80: 8 total, 8 up, 8 in 2024-09-17T00:04:53.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:53 smithi194 ceph-mon[51517]: pgmap v70: 97 pgs: 1 peering, 11 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 309 op/s; 181/1563 objects degraded (11.580%); 72 KiB/s, 23 objects/s recovering 2024-09-17T00:04:53.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:53 smithi194 ceph-mon[51517]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:53.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:53 smithi194 ceph-mon[51517]: osdmap e81: 8 total, 8 up, 8 in 2024-09-17T00:04:53.743 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:04:53.743 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (4m) 6s ago 13m 27.0M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:04:53.743 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (54s) 6s ago 13m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:04:53.743 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (48s) 43s ago 12m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:04:53.743 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (4m) 6s ago 12m 53.5M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:04:53.743 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (7m) 6s ago 7m 37.2M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (7m) 43s ago 7m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (3m) 6s ago 14m 496M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (2m) 43s ago 12m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (2m) 6s ago 14m 46.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (102s) 43s ago 12m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (6m) 6s ago 6m 259M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (6m) 43s ago 6m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (3m) 6s ago 12m 16.0M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (3m) 43s ago 12m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (16s) 6s ago 11m 11.9M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (11m) 6s ago 11m 473M 1937M 17.2.0 e1d6a67b021e 6e925e3dcd96 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (10m) 6s ago 10m 395M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:04:53.744 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (10m) 6s ago 10m 480M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:04:53.745 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (9m) 43s ago 9m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:04:53.745 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (9m) 43s ago 9m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:04:53.745 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (9m) 43s ago 9m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:04:53.745 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (8m) 43s ago 8m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:04:53.745 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (2m) 6s ago 12m 62.0M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:04:54.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 157094 9.27 MB/sec execute 278 sec latency 748.560 ms 2024-09-17T00:04:54.292 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:54.292 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.292 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.292 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:54.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:04:54.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:54.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:54.293 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:54.294 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:54.294 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:04:54.294 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:04:54.294 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-17T00:04:54.294 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:54.294 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:54 smithi007 ceph-mon[65487]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T00:04:54.352 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:04:54.352 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:04:54.353 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:04:54.353 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:54.353 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:04:54.353 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:04:54.353 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:54.354 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:04:54.354 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T00:04:54.354 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:04:54.354 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:54.354 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:04:54.354 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:04:54.355 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:04:54.355 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:04:54.355 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-17T00:04:54.355 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T00:04:54.355 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:04:54.356 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:04:54.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:54.660 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:54 smithi194 ceph-mon[51517]: osdmap e82: 8 total, 8 up, 8 in 2024-09-17T00:04:54.803 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:04:54.803 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:04:54.803 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:04:54.803 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:04:54.803 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "7/23 daemons upgraded", 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:04:54.804 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:04:55.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 157264 9.24 MB/sec execute 279 sec latency 1009.187 ms 2024-09-17T00:04:55.310 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:55 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/1509961676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:55.310 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:55 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 181/1563 objects degraded (11.580%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:55.310 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:55 smithi007 ceph-mon[65487]: pgmap v73: 97 pgs: 1 peering, 11 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 623 KiB/s wr, 56 op/s; 181/1563 objects degraded (11.580%); 0 B/s, 11 objects/s recovering 2024-09-17T00:04:55.310 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:55.310 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:55.311 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:55 smithi007 ceph-mon[65487]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T00:04:55.353 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_WARN Degraded data redundancy: 181/1563 objects degraded (11.580%), 11 pgs degraded 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 181/1563 objects degraded (11.580%), 11 pgs degraded 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.0 is active+recovery_wait+undersized+degraded+remapped, acting [7,1] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.c is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.f is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.10 is active+recovery_wait+undersized+degraded+remapped, acting [2,1] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1 is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.3 is active+recovery_wait+undersized+degraded+remapped, acting [4,6] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.b is active+recovery_wait+undersized+degraded+remapped, acting [3,4] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.f is active+recovery_wait+undersized+degraded+remapped, acting [7,4] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.10 is active+recovery_wait+undersized+degraded+remapped, acting [6,5] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2024-09-17T00:04:55.354 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1b is active+recovery_wait+undersized+degraded+remapped, acting [4,7] 2024-09-17T00:04:55.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:55 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/1509961676' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:04:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:55 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 181/1563 objects degraded (11.580%), 11 pgs degraded (PG_DEGRADED) 2024-09-17T00:04:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:55 smithi194 ceph-mon[51517]: pgmap v73: 97 pgs: 1 peering, 11 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 623 KiB/s wr, 56 op/s; 181/1563 objects degraded (11.580%); 0 B/s, 11 objects/s recovering 2024-09-17T00:04:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:04:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:04:55.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:55 smithi194 ceph-mon[51517]: osdmap e83: 8 total, 8 up, 8 in 2024-09-17T00:04:56.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 157862 9.26 MB/sec execute 280 sec latency 28.385 ms 2024-09-17T00:04:56.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:56 smithi007 ceph-mon[65487]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:56.323 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:56 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2582889984' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:04:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:56 smithi194 ceph-mon[51517]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:04:56.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:56 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2582889984' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:04:57.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 158512 9.27 MB/sec execute 281 sec latency 27.772 ms 2024-09-17T00:04:57.330 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:57 smithi007 ceph-mon[65487]: pgmap v75: 97 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 2.5 MiB/s wr, 198 op/s; 74/1524 objects degraded (4.856%); 3.5 MiB/s, 23 objects/s recovering 2024-09-17T00:04:57.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:57 smithi194 ceph-mon[51517]: pgmap v75: 97 pgs: 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 93 active+clean; 325 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 2.5 MiB/s wr, 198 op/s; 74/1524 objects degraded (4.856%); 3.5 MiB/s, 23 objects/s recovering 2024-09-17T00:04:58.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 158990 9.30 MB/sec execute 282 sec latency 28.886 ms 2024-09-17T00:04:58.882 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:58 smithi007 ceph-mon[65487]: pgmap v76: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 348 op/s; 2.7 MiB/s, 31 objects/s recovering 2024-09-17T00:04:58.882 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:58 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/1524 objects degraded (4.856%), 3 pgs degraded) 2024-09-17T00:04:58.882 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:04:58 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:04:59.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:58 smithi194 ceph-mon[51517]: pgmap v76: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 348 op/s; 2.7 MiB/s, 31 objects/s recovering 2024-09-17T00:04:59.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:58 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 74/1524 objects degraded (4.856%), 3 pgs degraded) 2024-09-17T00:04:59.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:04:58 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:04:59.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 159456 9.34 MB/sec execute 283 sec latency 34.739 ms 2024-09-17T00:05:00.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 159797 9.35 MB/sec execute 284 sec latency 43.236 ms 2024-09-17T00:05:01.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 160057 9.34 MB/sec execute 285 sec latency 35.153 ms 2024-09-17T00:05:01.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:01 smithi007 ceph-mon[65487]: pgmap v77: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 662 op/s; 2.1 MiB/s, 25 objects/s recovering 2024-09-17T00:05:02.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:01 smithi194 ceph-mon[51517]: pgmap v77: 97 pgs: 97 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.9 MiB/s wr, 662 op/s; 2.1 MiB/s, 25 objects/s recovering 2024-09-17T00:05:02.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 160291 9.31 MB/sec execute 286 sec latency 48.955 ms 2024-09-17T00:05:03.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 160518 9.29 MB/sec execute 287 sec latency 54.054 ms 2024-09-17T00:05:03.753 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:03 smithi007 ceph-mon[65487]: pgmap v78: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.5 MiB/s wr, 628 op/s; 1.9 MiB/s, 22 objects/s recovering 2024-09-17T00:05:04.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:03 smithi194 ceph-mon[51517]: pgmap v78: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.5 MiB/s wr, 628 op/s; 1.9 MiB/s, 22 objects/s recovering 2024-09-17T00:05:04.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 160757 9.28 MB/sec execute 288 sec latency 58.636 ms 2024-09-17T00:05:05.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 161107 9.28 MB/sec execute 289 sec latency 51.889 ms 2024-09-17T00:05:05.846 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:05 smithi007 ceph-mon[65487]: pgmap v79: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 535 op/s; 1.6 MiB/s, 19 objects/s recovering 2024-09-17T00:05:05.846 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:05 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:06.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:05 smithi194 ceph-mon[51517]: pgmap v79: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 5.6 MiB/s wr, 535 op/s; 1.6 MiB/s, 19 objects/s recovering 2024-09-17T00:05:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:05 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:06.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 161466 9.27 MB/sec execute 290 sec latency 73.881 ms 2024-09-17T00:05:07.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 161849 9.26 MB/sec execute 291 sec latency 50.287 ms 2024-09-17T00:05:07.910 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:07 smithi007 ceph-mon[65487]: pgmap v80: 97 pgs: 97 active+clean; 328 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.2 MiB/s wr, 646 op/s; 1.4 MiB/s, 16 objects/s recovering 2024-09-17T00:05:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:07 smithi194 ceph-mon[51517]: pgmap v80: 97 pgs: 97 active+clean; 328 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 9.8 KiB/s rd, 6.2 MiB/s wr, 646 op/s; 1.4 MiB/s, 16 objects/s recovering 2024-09-17T00:05:08.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 162216 9.26 MB/sec execute 292 sec latency 69.297 ms 2024-09-17T00:05:08.933 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:05:09.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:08 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:05:09.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 162632 9.30 MB/sec execute 293 sec latency 120.536 ms 2024-09-17T00:05:09.940 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:09 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:05:09.940 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:09 smithi007 ceph-mon[65487]: Upgrade: osd.1 is safe to restart 2024-09-17T00:05:09.940 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:09 smithi007 ceph-mon[65487]: pgmap v81: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 561 op/s; 0 B/s, 6 objects/s recovering 2024-09-17T00:05:09.940 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:09.940 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:09 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-17T00:05:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:09 smithi194 ceph-mon[51517]: Upgrade: osd.1 is safe to restart 2024-09-17T00:05:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:09 smithi194 ceph-mon[51517]: pgmap v81: 97 pgs: 97 active+clean; 327 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 561 op/s; 0 B/s, 6 objects/s recovering 2024-09-17T00:05:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:10.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 163113 9.33 MB/sec execute 294 sec latency 65.174 ms 2024-09-17T00:05:11.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 163442 9.34 MB/sec execute 295 sec latency 54.994 ms 2024-09-17T00:05:11.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:11 smithi007 ceph-mon[65487]: Upgrade: Updating osd.1 2024-09-17T00:05:11.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:11.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T00:05:11.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:05:11.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:11 smithi007 ceph-mon[65487]: Deploying daemon osd.1 on smithi007 2024-09-17T00:05:11.709 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:11 smithi007 ceph-mon[65487]: pgmap v82: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-09-17T00:05:11.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:11 smithi194 ceph-mon[51517]: Upgrade: Updating osd.1 2024-09-17T00:05:11.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:11.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-17T00:05:11.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:05:11.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:11 smithi194 ceph-mon[51517]: Deploying daemon osd.1 on smithi007 2024-09-17T00:05:11.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:11 smithi194 ceph-mon[51517]: pgmap v82: 97 pgs: 97 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 5.7 MiB/s wr, 566 op/s 2024-09-17T00:05:12.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 163786 9.32 MB/sec execute 296 sec latency 69.724 ms 2024-09-17T00:05:13.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 164077 9.30 MB/sec execute 297 sec latency 69.676 ms 2024-09-17T00:05:13.751 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:13 smithi007 ceph-mon[65487]: pgmap v83: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 441 op/s 2024-09-17T00:05:14.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:13 smithi194 ceph-mon[51517]: pgmap v83: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 441 op/s 2024-09-17T00:05:14.286 INFO:teuthology.orchestra.run.smithi007.stdout: 5 164308 9.29 MB/sec execute 298 sec latency 98.034 ms 2024-09-17T00:05:15.287 INFO:teuthology.orchestra.run.smithi007.stdout: 5 164566 9.28 MB/sec execute 299 sec latency 98.347 ms 2024-09-17T00:05:15.542 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:15 smithi007 systemd[1]: Stopping Ceph osd.1 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:05:15.793 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:15 smithi007 ceph-mon[65487]: pgmap v84: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-17T00:05:16.079 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:16 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[38215]: 2024-09-17T00:05:16.041+0000 7f6582e9b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:05:16.079 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:16 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[38215]: 2024-09-17T00:05:16.041+0000 7f6582e9b700 -1 osd.1 83 *** Got signal Terminated *** 2024-09-17T00:05:16.079 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:16 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[38215]: 2024-09-17T00:05:16.041+0000 7f6582e9b700 -1 osd.1 83 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:05:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:15 smithi194 ceph-mon[51517]: pgmap v84: 97 pgs: 97 active+clean; 340 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-17T00:05:16.287 INFO:teuthology.orchestra.run.smithi007.stdout: 5 164912 9.28 MB/sec execute 300 sec latency 50.384 ms 2024-09-17T00:05:16.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:16 smithi007 ceph-mon[65487]: osd.1 marked itself down and dead 2024-09-17T00:05:17.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:16 smithi194 ceph-mon[51517]: osd.1 marked itself down and dead 2024-09-17T00:05:17.287 INFO:teuthology.orchestra.run.smithi007.stdout: 5 165064 9.25 MB/sec execute 301 sec latency 568.636 ms 2024-09-17T00:05:17.661 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:17 smithi007 podman[73583]: 2024-09-17 00:05:17.385119724 +0000 UTC m=+1.755250029 container died 6e925e3dcd96851d2d6c26c63bd68ae5dcae18a67b2b01f356f58c94b0a31242 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, ceph=True, release=754, GIT_BRANCH=HEAD, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T00:05:17.866 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:17 smithi007 ceph-mon[65487]: pgmap v85: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 591 op/s 2024-09-17T00:05:17.866 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:17 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:05:17.866 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:17 smithi007 ceph-mon[65487]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T00:05:17.867 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:17 smithi007 podman[73583]: 2024-09-17 00:05:17.771490422 +0000 UTC m=+2.141620719 container cleanup 6e925e3dcd96851d2d6c26c63bd68ae5dcae18a67b2b01f356f58c94b0a31242 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, distribution-scope=public, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, vendor=Red Hat, Inc.) 2024-09-17T00:05:17.867 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:17 smithi007 bash[73583]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1 2024-09-17T00:05:18.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:17 smithi194 ceph-mon[51517]: pgmap v85: 97 pgs: 97 active+clean; 330 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 5.5 MiB/s wr, 591 op/s 2024-09-17T00:05:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:17 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:05:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:17 smithi194 ceph-mon[51517]: osdmap e84: 8 total, 7 up, 8 in 2024-09-17T00:05:18.176 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:18 smithi007 podman[73601]: 2024-09-17 00:05:18.157983948 +0000 UTC m=+0.766836356 container remove 6e925e3dcd96851d2d6c26c63bd68ae5dcae18a67b2b01f356f58c94b0a31242 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.19.8, version=8, RELEASE=HEAD, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, architecture=x86_64, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, release=754) 2024-09-17T00:05:18.287 INFO:teuthology.orchestra.run.smithi007.stdout: 5 165309 9.24 MB/sec execute 302 sec latency 1057.910 ms 2024-09-17T00:05:18.683 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:18 smithi007 podman[73687]: 2024-09-17 00:05:18.52567493 +0000 UTC m=+0.018634047 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:05:18.683 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:18 smithi007 podman[73687]: 2024-09-17 00:05:18.64273463 +0000 UTC m=+0.135693755 container create 2a702abd7da9af64587a83491564217fd39db83377bdf9fb6608b0e6056e6d2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-09-17T00:05:18.934 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:18 smithi007 ceph-mon[65487]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T00:05:19.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:18 smithi194 ceph-mon[51517]: osdmap e85: 8 total, 7 up, 8 in 2024-09-17T00:05:19.192 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:18 smithi007 podman[73687]: 2024-09-17 00:05:18.9463932 +0000 UTC m=+0.439352335 container init 2a702abd7da9af64587a83491564217fd39db83377bdf9fb6608b0e6056e6d2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-17T00:05:19.193 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:18 smithi007 podman[73687]: 2024-09-17 00:05:18.960732539 +0000 UTC m=+0.453691672 container start 2a702abd7da9af64587a83491564217fd39db83377bdf9fb6608b0e6056e6d2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-17T00:05:19.193 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:19 smithi007 podman[73687]: 2024-09-17 00:05:19.122100774 +0000 UTC m=+0.615059935 container attach 2a702abd7da9af64587a83491564217fd39db83377bdf9fb6608b0e6056e6d2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, ceph=True) 2024-09-17T00:05:19.287 INFO:teuthology.orchestra.run.smithi007.stdout: 5 165745 9.24 MB/sec execute 303 sec latency 51.587 ms 2024-09-17T00:05:19.447 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:19 smithi007 podman[73687]: 2024-09-17 00:05:19.286336128 +0000 UTC m=+0.779295247 container died 2a702abd7da9af64587a83491564217fd39db83377bdf9fb6608b0e6056e6d2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux ) 2024-09-17T00:05:19.954 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:19 smithi007 ceph-mon[65487]: pgmap v88: 97 pgs: 4 peering, 12 stale+active+clean, 81 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 465 op/s 2024-09-17T00:05:19.954 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:19 smithi007 ceph-mon[65487]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T00:05:20.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:19 smithi194 ceph-mon[51517]: pgmap v88: 97 pgs: 4 peering, 12 stale+active+clean, 81 active+clean; 327 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 4.5 MiB/s wr, 465 op/s 2024-09-17T00:05:20.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:19 smithi194 ceph-mon[51517]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-17T00:05:20.210 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 podman[73709]: 2024-09-17 00:05:20.151702045 +0000 UTC m=+0.856667290 container remove 2a702abd7da9af64587a83491564217fd39db83377bdf9fb6608b0e6056e6d2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97) 2024-09-17T00:05:20.210 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-17T00:05:20.210 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 systemd[1]: Stopped Ceph osd.1 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:05:20.210 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 43.497s CPU time. 2024-09-17T00:05:20.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 166220 9.28 MB/sec execute 304 sec latency 44.681 ms 2024-09-17T00:05:20.716 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 systemd[1]: Starting Ceph osd.1 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:05:20.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:20 smithi007 ceph-mon[65487]: pgmap v89: 97 pgs: 13 active+undersized, 4 peering, 1 stale+active+clean, 18 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.5 MiB/s wr, 535 op/s; 150/1569 objects degraded (9.560%) 2024-09-17T00:05:20.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:20 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 150/1569 objects degraded (9.560%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:20.976 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 podman[73802]: 2024-09-17 00:05:20.761136932 +0000 UTC m=+0.023932460 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:05:20.977 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:20 smithi007 podman[73802]: 2024-09-17 00:05:20.873067031 +0000 UTC m=+0.135862554 container create b1a223c23c2685158cafa4a15f75f2bd1054e99fcc5d594ce4978ca520bdf285 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-17T00:05:21.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:20 smithi194 ceph-mon[51517]: pgmap v89: 97 pgs: 13 active+undersized, 4 peering, 1 stale+active+clean, 18 active+undersized+degraded, 61 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 4.5 MiB/s wr, 535 op/s; 150/1569 objects degraded (9.560%) 2024-09-17T00:05:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:20 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 150/1569 objects degraded (9.560%), 18 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:21.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 166561 9.30 MB/sec execute 305 sec latency 60.742 ms 2024-09-17T00:05:21.479 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 podman[73802]: 2024-09-17 00:05:21.277307755 +0000 UTC m=+0.540103296 container init b1a223c23c2685158cafa4a15f75f2bd1054e99fcc5d594ce4978ca520bdf285 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T00:05:21.479 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 podman[73802]: 2024-09-17 00:05:21.286822612 +0000 UTC m=+0.549618138 container start b1a223c23c2685158cafa4a15f75f2bd1054e99fcc5d594ce4978ca520bdf285 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T00:05:21.479 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 podman[73802]: 2024-09-17 00:05:21.436363449 +0000 UTC m=+0.699158985 container attach b1a223c23c2685158cafa4a15f75f2bd1054e99fcc5d594ce4978ca520bdf285 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-17T00:05:21.735 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:05:21.735 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 bash[73802]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:05:21.735 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:05:21.735 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:21 smithi007 bash[73802]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:05:22.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 166899 9.30 MB/sec execute 306 sec latency 52.911 ms 2024-09-17T00:05:22.757 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:22 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T00:05:22.757 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:22 smithi007 bash[73802]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T00:05:22.757 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:22 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T00:05:22.757 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:22 smithi007 bash[73802]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T00:05:23.011 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:22 smithi007 ceph-mon[65487]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T00:05:23.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:22 smithi194 ceph-mon[51517]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 bash[73802]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 bash[73802]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 bash[73802]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T00:05:23.268 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 bash[73802]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-17T00:05:23.269 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate[73816]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T00:05:23.269 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 bash[73802]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-17T00:05:23.269 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 podman[73802]: 2024-09-17 00:05:23.142085357 +0000 UTC m=+2.404880880 container died b1a223c23c2685158cafa4a15f75f2bd1054e99fcc5d594ce4978ca520bdf285 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T00:05:23.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 167154 9.29 MB/sec execute 307 sec latency 53.899 ms 2024-09-17T00:05:23.781 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:23 smithi007 ceph-mon[65487]: pgmap v90: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 578 op/s; 189/1656 objects degraded (11.413%) 2024-09-17T00:05:24.038 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:23 smithi007 podman[73954]: 2024-09-17 00:05:23.901402324 +0000 UTC m=+0.752648243 container remove b1a223c23c2685158cafa4a15f75f2bd1054e99fcc5d594ce4978ca520bdf285 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:05:24.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:23 smithi194 ceph-mon[51517]: pgmap v90: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.8 MiB/s wr, 578 op/s; 189/1656 objects degraded (11.413%) 2024-09-17T00:05:24.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 167399 9.27 MB/sec execute 308 sec latency 44.402 ms 2024-09-17T00:05:24.327 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:24 smithi007 podman[74011]: 2024-09-17 00:05:24.037770125 +0000 UTC m=+0.018261970 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:05:24.327 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:24 smithi007 podman[74011]: 2024-09-17 00:05:24.196226257 +0000 UTC m=+0.176718088 container create 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:05:24.651 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:24 smithi007 podman[74011]: 2024-09-17 00:05:24.581650784 +0000 UTC m=+0.562142623 container init 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True) 2024-09-17T00:05:24.651 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:24 smithi007 podman[74011]: 2024-09-17 00:05:24.591282345 +0000 UTC m=+0.571774183 container start 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-17T00:05:24.759 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:24 smithi007 bash[74011]: 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 2024-09-17T00:05:24.759 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:24 smithi007 systemd[1]: Started Ceph osd.1 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:05:25.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 167742 9.26 MB/sec execute 309 sec latency 56.308 ms 2024-09-17T00:05:25.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:25 smithi007 ceph-mon[65487]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.6 MiB/s wr, 307 op/s; 189/1656 objects degraded (11.413%) 2024-09-17T00:05:25.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:25.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:25.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:25.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:25.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:05:25.950 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:05:26.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:25 smithi194 ceph-mon[51517]: pgmap v91: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 3.6 MiB/s wr, 307 op/s; 189/1656 objects degraded (11.413%) 2024-09-17T00:05:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:05:26.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 168145 9.27 MB/sec execute 310 sec latency 37.624 ms 2024-09-17T00:05:26.941 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:26 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 183/1611 objects degraded (11.359%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:26.941 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:26 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:05:26.740+0000 7f5771282640 -1 Falling back to public interface 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (4m) 39s ago 13m 27.0M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (87s) 39s ago 13m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (81s) 76s ago 13m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (4m) 39s ago 13m 53.5M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (7m) 39s ago 7m 37.2M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (7m) 76s ago 7m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (3m) 39s ago 14m 496M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (3m) 76s ago 13m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:05:27.041 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (2m) 39s ago 15m 46.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (2m) 76s ago 12m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (7m) 39s ago 7m 259M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (7m) 76s ago 7m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (4m) 39s ago 13m 16.0M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (4m) 76s ago 12m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (49s) 39s ago 12m 11.9M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 starting - - - 1937M 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (11m) 39s ago 11m 395M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (10m) 39s ago 10m 480M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (10m) 76s ago 10m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (10m) 76s ago 10m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (9m) 76s ago 9m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (9m) 76s ago 9m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:05:27.042 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (3m) 39s ago 12m 62.0M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:05:27.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:26 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 183/1611 objects degraded (11.359%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:27.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 168532 9.26 MB/sec execute 311 sec latency 84.243 ms 2024-09-17T00:05:27.678 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:05:27.678 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:05:27.678 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 1 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T00:05:27.679 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T00:05:27.680 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:05:27.680 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:05:27.975 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:27 smithi007 ceph-mon[65487]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:27.975 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:27 smithi007 ceph-mon[65487]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:27.975 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:27 smithi007 ceph-mon[65487]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 474 op/s; 183/1611 objects degraded (11.359%) 2024-09-17T00:05:27.975 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:27 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2876991620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:28.145 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "7/23 daemons upgraded", 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:05:28.146 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:05:28.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:27 smithi194 ceph-mon[51517]: from='client.44161 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:27 smithi194 ceph-mon[51517]: from='client.34232 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:27 smithi194 ceph-mon[51517]: pgmap v92: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 339 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 474 op/s; 183/1611 objects degraded (11.359%) 2024-09-17T00:05:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:27 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2876991620' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:28.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 168951 9.25 MB/sec execute 312 sec latency 62.572 ms 2024-09-17T00:05:28.689 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 183/1611 objects degraded (11.359%), 24 pgs degraded 2024-09-17T00:05:28.689 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T00:05:28.689 INFO:teuthology.orchestra.run.smithi007.stdout: osd.1 (root=default,host=smithi007) is down 2024-09-17T00:05:28.689 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 183/1611 objects degraded (11.359%), 24 pgs degraded 2024-09-17T00:05:28.689 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.0 is active+undersized+degraded, acting [7,0] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.6 is active+undersized+degraded, acting [6,4] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.8 is active+undersized+degraded, acting [7,0] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.9 is active+undersized+degraded, acting [7,3] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.a is active+undersized+degraded, acting [3,7] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.d is active+undersized+degraded, acting [4,3] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.10 is active+undersized+degraded, acting [2,0] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.11 is active+undersized+degraded, acting [6,4] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.15 is active+undersized+degraded, acting [5,0] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.0 is active+undersized+degraded, acting [2,6] 2024-09-17T00:05:28.690 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.4 is active+undersized+degraded, acting [2,5] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.8 is active+undersized+degraded, acting [3,7] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.a is active+undersized+degraded, acting [6,4] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.e is active+undersized+degraded, acting [7,4] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.16 is active+undersized+degraded, acting [5,7] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.18 is active+undersized+degraded, acting [3,0] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.19 is active+undersized+degraded, acting [3,4] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1a is active+undersized+degraded, acting [4,2] 2024-09-17T00:05:28.691 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-09-17T00:05:28.692 INFO:teuthology.orchestra.run.smithi007.stdout: pg 4.a is active+undersized+degraded, acting [6,7] 2024-09-17T00:05:28.692 INFO:teuthology.orchestra.run.smithi007.stdout: pg 4.14 is active+undersized+degraded, acting [3,7] 2024-09-17T00:05:28.986 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:28 smithi007 ceph-mon[65487]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:28.986 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:28 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/361868238' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:05:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:28 smithi194 ceph-mon[51517]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:28 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/361868238' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:05:29.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 169239 9.25 MB/sec execute 313 sec latency 41.382 ms 2024-09-17T00:05:30.009 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:29 smithi007 ceph-mon[65487]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:30.009 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:29 smithi007 ceph-mon[65487]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 185/1587 objects degraded (11.657%) 2024-09-17T00:05:30.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:29 smithi194 ceph-mon[51517]: from='client.34244 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:05:30.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:29 smithi194 ceph-mon[51517]: pgmap v93: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 427 op/s; 185/1587 objects degraded (11.657%) 2024-09-17T00:05:30.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 169601 9.27 MB/sec execute 314 sec latency 35.786 ms 2024-09-17T00:05:31.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 169908 9.27 MB/sec execute 315 sec latency 63.258 ms 2024-09-17T00:05:31.806 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:31 smithi007 ceph-mon[65487]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 515 op/s; 181/1581 objects degraded (11.448%) 2024-09-17T00:05:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:31 smithi194 ceph-mon[51517]: pgmap v94: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 333 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.3 MiB/s wr, 515 op/s; 181/1581 objects degraded (11.448%) 2024-09-17T00:05:32.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 170267 9.30 MB/sec execute 316 sec latency 41.221 ms 2024-09-17T00:05:32.816 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:32 smithi007 ceph-mon[65487]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 446 op/s; 182/1599 objects degraded (11.382%) 2024-09-17T00:05:32.816 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:32 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 182/1599 objects degraded (11.382%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:33.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:32 smithi194 ceph-mon[51517]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 446 op/s; 182/1599 objects degraded (11.382%) 2024-09-17T00:05:33.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:32 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 182/1599 objects degraded (11.382%), 24 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:33.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 170530 9.29 MB/sec execute 317 sec latency 88.135 ms 2024-09-17T00:05:33.826 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:33 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:05:33.588+0000 7f5771282640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-17T00:05:34.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 170725 9.27 MB/sec execute 318 sec latency 59.783 ms 2024-09-17T00:05:34.590 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:34 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:34.590 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:34 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:34.842 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:34 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:05:34.708+0000 7f5771282640 -1 osd.1 83 log_to_monitors true 2024-09-17T00:05:34.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:34 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:34 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:35.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 170973 9.25 MB/sec execute 319 sec latency 47.331 ms 2024-09-17T00:05:35.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:35 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:35.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:35 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:35.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:35 smithi007 ceph-mon[65487]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 417 op/s; 182/1599 objects degraded (11.382%) 2024-09-17T00:05:35.604 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:35 smithi007 ceph-mon[65487]: from='osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T00:05:35.855 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:05:35 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:05:35.610+0000 7f576902d640 -1 osd.1 83 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:05:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:35 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:35 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:35 smithi194 ceph-mon[51517]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 336 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.1 MiB/s wr, 417 op/s; 182/1599 objects degraded (11.382%) 2024-09-17T00:05:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:35 smithi194 ceph-mon[51517]: from='osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-17T00:05:36.297 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171283 9.24 MB/sec execute 320 sec latency 59.143 ms 2024-09-17T00:05:36.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:36 smithi007 ceph-mon[65487]: from='osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T00:05:36.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:36 smithi007 ceph-mon[65487]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T00:05:36.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:36 smithi007 ceph-mon[65487]: from='osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:05:36.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:36 smithi194 ceph-mon[51517]: from='osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-17T00:05:36.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:36 smithi194 ceph-mon[51517]: osdmap e86: 8 total, 7 up, 8 in 2024-09-17T00:05:36.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:36 smithi194 ceph-mon[51517]: from='osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:05:37.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171341 9.22 MB/sec execute 321 sec latency 767.303 ms 2024-09-17T00:05:37.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:37 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:05:37.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:37 smithi007 ceph-mon[65487]: osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813] boot 2024-09-17T00:05:37.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:37 smithi007 ceph-mon[65487]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T00:05:37.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:37 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:05:37.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:37 smithi007 ceph-mon[65487]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 562 op/s; 188/1647 objects degraded (11.415%) 2024-09-17T00:05:37.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:37 smithi007 ceph-mon[65487]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T00:05:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:37 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:05:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:37 smithi194 ceph-mon[51517]: osd.1 [v2:172.21.15.7:6810/2470999813,v1:172.21.15.7:6811/2470999813] boot 2024-09-17T00:05:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:37 smithi194 ceph-mon[51517]: osdmap e87: 8 total, 8 up, 8 in 2024-09-17T00:05:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:37 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-17T00:05:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:37 smithi194 ceph-mon[51517]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 562 op/s; 188/1647 objects degraded (11.415%) 2024-09-17T00:05:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:37 smithi194 ceph-mon[51517]: osdmap e88: 8 total, 8 up, 8 in 2024-09-17T00:05:38.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171341 9.19 MB/sec execute 322 sec latency 1767.335 ms 2024-09-17T00:05:39.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171419 9.17 MB/sec execute 323 sec latency 2388.783 ms 2024-09-17T00:05:39.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:39 smithi007 ceph-mon[65487]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T00:05:39.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:39 smithi007 ceph-mon[65487]: pgmap v102: 97 pgs: 3 peering, 13 active+undersized, 23 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 500 op/s; 173/1647 objects degraded (10.504%) 2024-09-17T00:05:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:39 smithi194 ceph-mon[51517]: osdmap e89: 8 total, 8 up, 8 in 2024-09-17T00:05:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:39 smithi194 ceph-mon[51517]: pgmap v102: 97 pgs: 3 peering, 13 active+undersized, 23 active+undersized+degraded, 58 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.5 MiB/s wr, 500 op/s; 173/1647 objects degraded (10.504%) 2024-09-17T00:05:40.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171508 9.14 MB/sec execute 324 sec latency 744.042 ms 2024-09-17T00:05:40.836 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:40 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 173/1647 objects degraded (10.504%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:40.836 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:40 smithi007 ceph-mon[65487]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T00:05:40.836 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:40.836 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:40.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:40 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 173/1647 objects degraded (10.504%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:40 smithi194 ceph-mon[51517]: osdmap e90: 8 total, 8 up, 8 in 2024-09-17T00:05:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:41.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171702 9.13 MB/sec execute 325 sec latency 1049.310 ms 2024-09-17T00:05:41.837 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T00:05:41.837 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.837 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: pgmap v105: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 84 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.1 MiB/s wr, 225 op/s; 115/1500 objects degraded (7.667%); 2.8 MiB/s, 43 keys/s, 19 objects/s recovering 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:41.838 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: osdmap e91: 8 total, 8 up, 8 in 2024-09-17T00:05:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:05:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:05:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: pgmap v105: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+degraded, 84 active+clean; 333 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.1 MiB/s wr, 225 op/s; 115/1500 objects degraded (7.667%); 2.8 MiB/s, 43 keys/s, 19 objects/s recovering 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:05:41.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:42.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 171830 9.12 MB/sec execute 326 sec latency 920.211 ms 2024-09-17T00:05:42.860 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:42 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:42.860 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:42 smithi007 ceph-mon[65487]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T00:05:42.860 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:42 smithi007 ceph-mon[65487]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T00:05:42.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:42 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:42 smithi194 ceph-mon[51517]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-09-17T00:05:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:42 smithi194 ceph-mon[51517]: osdmap e92: 8 total, 8 up, 8 in 2024-09-17T00:05:43.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 172129 9.10 MB/sec execute 327 sec latency 1013.503 ms 2024-09-17T00:05:43.894 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:43 smithi007 ceph-mon[65487]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T00:05:43.894 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:43 smithi007 ceph-mon[65487]: pgmap v108: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 89 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.6 MiB/s wr, 216 op/s; 93/1380 objects degraded (6.739%); 3.9 MiB/s, 43 keys/s, 30 objects/s recovering 2024-09-17T00:05:43.894 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:43 smithi007 ceph-mon[65487]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T00:05:43.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:43 smithi194 ceph-mon[51517]: osdmap e93: 8 total, 8 up, 8 in 2024-09-17T00:05:43.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:43 smithi194 ceph-mon[51517]: pgmap v108: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 89 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.6 MiB/s wr, 216 op/s; 93/1380 objects degraded (6.739%); 3.9 MiB/s, 43 keys/s, 30 objects/s recovering 2024-09-17T00:05:43.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:43 smithi194 ceph-mon[51517]: osdmap e94: 8 total, 8 up, 8 in 2024-09-17T00:05:44.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 172462 9.09 MB/sec execute 328 sec latency 73.017 ms 2024-09-17T00:05:44.669 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:44 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 93/1380 objects degraded (6.739%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:45.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:44 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 93/1380 objects degraded (6.739%), 5 pgs degraded (PG_DEGRADED) 2024-09-17T00:05:45.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 172714 9.07 MB/sec execute 329 sec latency 46.937 ms 2024-09-17T00:05:45.681 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:45 smithi007 ceph-mon[65487]: pgmap v110: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 89 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 984 KiB/s wr, 85 op/s; 93/1380 objects degraded (6.739%); 3.2 MiB/s, 24 objects/s recovering 2024-09-17T00:05:46.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:45 smithi194 ceph-mon[51517]: pgmap v110: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 89 active+clean; 328 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 984 KiB/s wr, 85 op/s; 93/1380 objects degraded (6.739%); 3.2 MiB/s, 24 objects/s recovering 2024-09-17T00:05:46.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 173032 9.09 MB/sec execute 330 sec latency 46.689 ms 2024-09-17T00:05:46.963 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:46 smithi007 ceph-mon[65487]: pgmap v111: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.3 MiB/s wr, 301 op/s; 1.4 MiB/s, 26 objects/s recovering 2024-09-17T00:05:46.963 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:46 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 93/1380 objects degraded (6.739%), 5 pgs degraded) 2024-09-17T00:05:46.963 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:46 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:05:47.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:46 smithi194 ceph-mon[51517]: pgmap v111: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 2.3 MiB/s wr, 301 op/s; 1.4 MiB/s, 26 objects/s recovering 2024-09-17T00:05:47.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:46 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 93/1380 objects degraded (6.739%), 5 pgs degraded) 2024-09-17T00:05:47.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:46 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:05:47.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 173312 9.10 MB/sec execute 331 sec latency 52.984 ms 2024-09-17T00:05:48.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 173743 9.12 MB/sec execute 332 sec latency 70.508 ms 2024-09-17T00:05:49.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 174044 9.13 MB/sec execute 333 sec latency 89.578 ms 2024-09-17T00:05:49.775 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:49 smithi007 ceph-mon[65487]: pgmap v112: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 2.2 MiB/s wr, 268 op/s; 614 KiB/s, 18 objects/s recovering 2024-09-17T00:05:50.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:49 smithi194 ceph-mon[51517]: pgmap v112: 97 pgs: 97 active+clean; 330 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 2.2 MiB/s wr, 268 op/s; 614 KiB/s, 18 objects/s recovering 2024-09-17T00:05:50.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 174351 9.12 MB/sec execute 334 sec latency 59.737 ms 2024-09-17T00:05:51.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 174530 9.09 MB/sec execute 335 sec latency 112.482 ms 2024-09-17T00:05:51.823 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:51 smithi007 ceph-mon[65487]: pgmap v113: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 461 op/s; 507 KiB/s, 13 objects/s recovering 2024-09-17T00:05:52.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:51 smithi194 ceph-mon[51517]: pgmap v113: 97 pgs: 97 active+clean; 343 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 4.2 MiB/s wr, 461 op/s; 507 KiB/s, 13 objects/s recovering 2024-09-17T00:05:52.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 174709 9.07 MB/sec execute 336 sec latency 80.555 ms 2024-09-17T00:05:53.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 174933 9.06 MB/sec execute 337 sec latency 72.767 ms 2024-09-17T00:05:53.897 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:53 smithi007 ceph-mon[65487]: pgmap v114: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 MiB/s wr, 402 op/s; 409 KiB/s, 11 objects/s recovering 2024-09-17T00:05:54.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:53 smithi194 ceph-mon[51517]: pgmap v114: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.3 MiB/s wr, 402 op/s; 409 KiB/s, 11 objects/s recovering 2024-09-17T00:05:54.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 175168 9.06 MB/sec execute 338 sec latency 85.008 ms 2024-09-17T00:05:55.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 175697 9.06 MB/sec execute 339 sec latency 45.206 ms 2024-09-17T00:05:55.926 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:55 smithi007 ceph-mon[65487]: pgmap v115: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 370 KiB/s, 10 objects/s recovering 2024-09-17T00:05:55.926 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:55.926 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:55.926 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:55 smithi194 ceph-mon[51517]: pgmap v115: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.9 MiB/s wr, 364 op/s; 370 KiB/s, 10 objects/s recovering 2024-09-17T00:05:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:05:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:56.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 176177 9.06 MB/sec execute 340 sec latency 48.973 ms 2024-09-17T00:05:57.046 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:56 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:56 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:57.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 176438 9.06 MB/sec execute 341 sec latency 89.765 ms 2024-09-17T00:05:57.801 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:57 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:57.801 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:57 smithi007 ceph-mon[65487]: Upgrade: osd.2 is safe to restart 2024-09-17T00:05:57.801 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:57 smithi007 ceph-mon[65487]: pgmap v116: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 509 op/s; 341 KiB/s, 9 objects/s recovering 2024-09-17T00:05:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:57 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-17T00:05:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:57 smithi194 ceph-mon[51517]: Upgrade: osd.2 is safe to restart 2024-09-17T00:05:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:57 smithi194 ceph-mon[51517]: pgmap v116: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 509 op/s; 341 KiB/s, 9 objects/s recovering 2024-09-17T00:05:58.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 176802 9.08 MB/sec execute 342 sec latency 58.455 ms 2024-09-17T00:05:59.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 177201 9.10 MB/sec execute 343 sec latency 81.716 ms 2024-09-17T00:05:59.302 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:05:59.841 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:59 smithi007 ceph-mon[65487]: Upgrade: Updating osd.2 2024-09-17T00:05:59.841 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:05:59.841 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:59 smithi007 ceph-mon[65487]: pgmap v117: 97 pgs: 97 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-17T00:05:59.841 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T00:05:59.841 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:05:59.841 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:05:59 smithi007 ceph-mon[65487]: Deploying daemon osd.2 on smithi007 2024-09-17T00:06:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:59 smithi194 ceph-mon[51517]: Upgrade: Updating osd.2 2024-09-17T00:06:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:59 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:59 smithi194 ceph-mon[51517]: pgmap v117: 97 pgs: 97 active+clean; 331 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 384 op/s 2024-09-17T00:06:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:59 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-17T00:06:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:59 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:06:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:05:59 smithi194 ceph-mon[51517]: Deploying daemon osd.2 on smithi007 2024-09-17T00:06:00.270 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:06:00.270 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (5m) 26s ago 14m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:06:00.270 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (2m) 26s ago 14m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (114s) 110s ago 13m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (5m) 26s ago 14m 53.9M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (8m) 26s ago 8m 37.4M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (8m) 110s ago 8m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (4m) 26s ago 15m 500M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (3m) 110s ago 13m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (3m) 26s ago 15m 48.7M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (2m) 110s ago 13m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (7m) 26s ago 7m 253M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (7m) 110s ago 7m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (5m) 26s ago 13m 15.8M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:06:00.271 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (4m) 110s ago 13m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (82s) 26s ago 12m 216M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (36s) 26s ago 12m 11.3M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (11m) 26s ago 11m 437M 1937M 17.2.0 e1d6a67b021e 68af761db0ed 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (11m) 26s ago 11m 515M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (11m) 110s ago 10m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (10m) 110s ago 10m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (10m) 110s ago 10m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (9m) 110s ago 9m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:06:00.272 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (3m) 26s ago 13m 63.5M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:06:00.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 177464 9.10 MB/sec execute 344 sec latency 82.157 ms 2024-09-17T00:06:00.859 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:00 smithi007 ceph-mon[65487]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:00.859 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:00 smithi007 ceph-mon[65487]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:00.901 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:06:00.902 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:06:01.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:00 smithi194 ceph-mon[51517]: from='client.44183 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:01.060 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:00 smithi194 ceph-mon[51517]: from='client.34258 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:01.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 177711 9.10 MB/sec execute 345 sec latency 42.696 ms 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "8/23 daemons upgraded", 2024-09-17T00:06:01.458 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:06:01.459 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:06:01.459 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:06:01.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:01 smithi007 ceph-mon[65487]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:01.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:01 smithi007 ceph-mon[65487]: pgmap v118: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 536 op/s 2024-09-17T00:06:01.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:01 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/804171731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:02.100 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:06:02.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:01 smithi194 ceph-mon[51517]: from='client.34262 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:02.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:01 smithi194 ceph-mon[51517]: pgmap v118: 97 pgs: 97 active+clean; 334 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.2 MiB/s wr, 536 op/s 2024-09-17T00:06:02.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:01 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/804171731' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:02.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 177957 9.08 MB/sec execute 346 sec latency 50.354 ms 2024-09-17T00:06:02.904 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:02 smithi007 ceph-mon[65487]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:02.904 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:02 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2294636171' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:06:03.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:02 smithi194 ceph-mon[51517]: from='client.34270 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:02 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2294636171' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:06:03.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 178172 9.06 MB/sec execute 347 sec latency 83.570 ms 2024-09-17T00:06:03.661 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:03 smithi007 systemd[1]: Stopping Ceph osd.2 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:06:03.912 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:03 smithi007 ceph-mon[65487]: pgmap v119: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 405 op/s 2024-09-17T00:06:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:03 smithi194 ceph-mon[51517]: pgmap v119: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 405 op/s 2024-09-17T00:06:04.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 178441 9.05 MB/sec execute 348 sec latency 65.994 ms 2024-09-17T00:06:04.429 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:04 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[41865]: 2024-09-17T00:06:04.286+0000 7f69cfdcf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:06:04.429 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:04 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[41865]: 2024-09-17T00:06:04.286+0000 7f69cfdcf700 -1 osd.2 94 *** Got signal Terminated *** 2024-09-17T00:06:04.429 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:04 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[41865]: 2024-09-17T00:06:04.286+0000 7f69cfdcf700 -1 osd.2 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:06:05.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:04 smithi007 ceph-mon[65487]: osd.2 marked itself down and dead 2024-09-17T00:06:05.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:04 smithi194 ceph-mon[51517]: osd.2 marked itself down and dead 2024-09-17T00:06:05.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 178605 9.04 MB/sec execute 349 sec latency 545.457 ms 2024-09-17T00:06:05.639 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:05 smithi007 podman[76289]: 2024-09-17 00:06:05.38041374 +0000 UTC m=+1.502087161 container died 68af761db0edc7704137a61e60c2e7f3488dbd92ec5d0dc6e4d1946c610c696a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, architecture=x86_64, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., release=754, CEPH_POINT_RELEASE=-17.2.0, ceph=True, maintainer=Guillaume Abrioux , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream) 2024-09-17T00:06:05.897 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:05 smithi007 ceph-mon[65487]: pgmap v120: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 380 op/s 2024-09-17T00:06:05.897 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:05 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:06:05.897 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:05 smithi007 ceph-mon[65487]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T00:06:05.898 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:05 smithi007 podman[76289]: 2024-09-17 00:06:05.753671073 +0000 UTC m=+1.875344485 container cleanup 68af761db0edc7704137a61e60c2e7f3488dbd92ec5d0dc6e4d1946c610c696a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, GIT_BRANCH=HEAD, architecture=x86_64, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, version=8, name=centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, RELEASE=HEAD) 2024-09-17T00:06:05.898 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:05 smithi007 bash[76289]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2 2024-09-17T00:06:06.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:05 smithi194 ceph-mon[51517]: pgmap v120: 97 pgs: 97 active+clean; 339 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 380 op/s 2024-09-17T00:06:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:05 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:06:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:05 smithi194 ceph-mon[51517]: osdmap e95: 8 total, 7 up, 8 in 2024-09-17T00:06:06.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 178699 9.02 MB/sec execute 350 sec latency 1068.479 ms 2024-09-17T00:06:06.339 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:06 smithi007 podman[76309]: 2024-09-17 00:06:06.207253525 +0000 UTC m=+0.809191398 container remove 68af761db0edc7704137a61e60c2e7f3488dbd92ec5d0dc6e4d1946c610c696a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, io.openshift.expose-services=, ceph=True, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., architecture=x86_64, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD) 2024-09-17T00:06:06.840 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:06 smithi007 ceph-mon[65487]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T00:06:06.840 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:06 smithi007 podman[76433]: 2024-09-17 00:06:06.569831735 +0000 UTC m=+0.020212954 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:06:06.840 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:06 smithi007 podman[76433]: 2024-09-17 00:06:06.776070826 +0000 UTC m=+0.226452031 container create 3b5b2d15e87475b578219945e1d12d3e6eccb6c11a6f8a9697936e2f84effc39 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T00:06:07.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:06 smithi194 ceph-mon[51517]: osdmap e96: 8 total, 7 up, 8 in 2024-09-17T00:06:07.298 INFO:teuthology.orchestra.run.smithi007.stdout: 5 178926 9.00 MB/sec execute 351 sec latency 68.674 ms 2024-09-17T00:06:07.375 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:07 smithi007 podman[76433]: 2024-09-17 00:06:07.171872447 +0000 UTC m=+0.622253677 container init 3b5b2d15e87475b578219945e1d12d3e6eccb6c11a6f8a9697936e2f84effc39 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2) 2024-09-17T00:06:07.375 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:07 smithi007 podman[76433]: 2024-09-17 00:06:07.184948193 +0000 UTC m=+0.635329406 container start 3b5b2d15e87475b578219945e1d12d3e6eccb6c11a6f8a9697936e2f84effc39 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True) 2024-09-17T00:06:07.376 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:07 smithi007 podman[76433]: 2024-09-17 00:06:07.296856087 +0000 UTC m=+0.747237301 container attach 3b5b2d15e87475b578219945e1d12d3e6eccb6c11a6f8a9697936e2f84effc39 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97) 2024-09-17T00:06:07.634 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:07 smithi007 podman[76433]: 2024-09-17 00:06:07.50954965 +0000 UTC m=+0.959930863 container died 3b5b2d15e87475b578219945e1d12d3e6eccb6c11a6f8a9697936e2f84effc39 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T00:06:07.887 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:07 smithi007 ceph-mon[65487]: pgmap v123: 97 pgs: 24 peering, 73 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 498 op/s 2024-09-17T00:06:07.887 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:07 smithi007 ceph-mon[65487]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T00:06:08.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:07 smithi194 ceph-mon[51517]: pgmap v123: 97 pgs: 24 peering, 73 active+clean; 334 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.3 MiB/s wr, 498 op/s 2024-09-17T00:06:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:07 smithi194 ceph-mon[51517]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T00:06:08.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 179202 8.99 MB/sec execute 352 sec latency 61.970 ms 2024-09-17T00:06:08.649 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:08 smithi007 podman[76470]: 2024-09-17 00:06:08.417741229 +0000 UTC m=+0.892971875 container remove 3b5b2d15e87475b578219945e1d12d3e6eccb6c11a6f8a9697936e2f84effc39 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, ceph=True) 2024-09-17T00:06:08.649 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:08 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-17T00:06:08.649 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:08 smithi007 systemd[1]: Stopped Ceph osd.2 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:06:08.649 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:08 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 27.022s CPU time. 2024-09-17T00:06:08.900 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:08 smithi007 systemd[1]: Starting Ceph osd.2 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:06:08.900 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:08 smithi007 ceph-mon[65487]: pgmap v124: 97 pgs: 1 active+undersized, 24 peering, 1 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.4 MiB/s wr, 267 op/s; 12/1479 objects degraded (0.811%) 2024-09-17T00:06:09.158 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 podman[76567]: 2024-09-17 00:06:08.925245493 +0000 UTC m=+0.022117623 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:06:09.158 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 podman[76567]: 2024-09-17 00:06:09.05697138 +0000 UTC m=+0.153843489 container create 47b59fb1c85073d18f00434deae4875b02b71e31041fbb7b375606ef07a0f627 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T00:06:09.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:08 smithi194 ceph-mon[51517]: pgmap v124: 97 pgs: 1 active+undersized, 24 peering, 1 active+undersized+degraded, 71 active+clean; 332 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.4 MiB/s wr, 267 op/s; 12/1479 objects degraded (0.811%) 2024-09-17T00:06:09.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 179441 8.98 MB/sec execute 353 sec latency 72.844 ms 2024-09-17T00:06:09.411 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 podman[76567]: 2024-09-17 00:06:09.376659468 +0000 UTC m=+0.473531571 container init 47b59fb1c85073d18f00434deae4875b02b71e31041fbb7b375606ef07a0f627 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:06:09.411 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 podman[76567]: 2024-09-17 00:06:09.383327122 +0000 UTC m=+0.480199221 container start 47b59fb1c85073d18f00434deae4875b02b71e31041fbb7b375606ef07a0f627 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731) 2024-09-17T00:06:09.672 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 podman[76567]: 2024-09-17 00:06:09.527267359 +0000 UTC m=+0.624139740 container attach 47b59fb1c85073d18f00434deae4875b02b71e31041fbb7b375606ef07a0f627 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T00:06:09.672 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:09.672 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 bash[76567]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:09.672 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:09.672 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:09 smithi007 bash[76567]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:09.941 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:09 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 12/1479 objects degraded (0.811%), 1 pg degraded (PG_DEGRADED) 2024-09-17T00:06:09.941 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:09.941 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:09 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 12/1479 objects degraded (0.811%), 1 pg degraded (PG_DEGRADED) 2024-09-17T00:06:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:10.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 179666 8.97 MB/sec execute 354 sec latency 72.563 ms 2024-09-17T00:06:10.445 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T00:06:10.445 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T00:06:10.445 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T00:06:10.445 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T00:06:10.958 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-mon[65487]: pgmap v125: 97 pgs: 7 active+undersized, 2 peering, 17 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.5 MiB/s wr, 277 op/s; 130/1479 objects degraded (8.790%) 2024-09-17T00:06:10.958 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T00:06:10.958 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T00:06:10.958 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T00:06:10.958 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T00:06:10.958 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T00:06:10.959 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-17T00:06:10.959 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T00:06:10.959 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-17T00:06:10.959 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate[76582]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T00:06:10.959 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 bash[76567]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-17T00:06:10.959 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:10 smithi007 podman[76567]: 2024-09-17 00:06:10.871956951 +0000 UTC m=+1.968829049 container died 47b59fb1c85073d18f00434deae4875b02b71e31041fbb7b375606ef07a0f627 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T00:06:11.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:10 smithi194 ceph-mon[51517]: pgmap v125: 97 pgs: 7 active+undersized, 2 peering, 17 active+undersized+degraded, 71 active+clean; 331 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 2.5 MiB/s wr, 277 op/s; 130/1479 objects degraded (8.790%) 2024-09-17T00:06:11.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 179955 8.97 MB/sec execute 355 sec latency 69.648 ms 2024-09-17T00:06:11.774 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:11 smithi007 podman[76721]: 2024-09-17 00:06:11.657651176 +0000 UTC m=+0.776833733 container remove 47b59fb1c85073d18f00434deae4875b02b71e31041fbb7b375606ef07a0f627 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T00:06:12.031 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:11 smithi007 ceph-mon[65487]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-17T00:06:12.032 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:11 smithi007 podman[76777]: 2024-09-17 00:06:11.787859151 +0000 UTC m=+0.021446358 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:06:12.032 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:11 smithi007 podman[76777]: 2024-09-17 00:06:11.903907905 +0000 UTC m=+0.137495098 container create 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-17T00:06:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:11 smithi194 ceph-mon[51517]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2024-09-17T00:06:12.285 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:12 smithi007 podman[76777]: 2024-09-17 00:06:12.21557267 +0000 UTC m=+0.449159886 container init 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-17T00:06:12.285 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:12 smithi007 podman[76777]: 2024-09-17 00:06:12.225348363 +0000 UTC m=+0.458935557 container start 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T00:06:12.286 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:12 smithi007 ceph-osd[76795]: -- 172.21.15.7:0/700682933 <== mon.0 v2:172.21.15.7:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e2fbba01e0 con 0x55e2fbb76000 2024-09-17T00:06:12.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 180227 8.97 MB/sec execute 356 sec latency 50.379 ms 2024-09-17T00:06:12.577 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:12 smithi007 bash[76777]: 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 2024-09-17T00:06:12.577 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:12 smithi007 systemd[1]: Started Ceph osd.2 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:06:13.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 180454 8.98 MB/sec execute 357 sec latency 48.151 ms 2024-09-17T00:06:13.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:13.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:13.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:06:13.619 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:13 smithi007 ceph-mon[65487]: pgmap v126: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.6 MiB/s wr, 443 op/s; 148/1530 objects degraded (9.673%) 2024-09-17T00:06:13.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:06:13.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:13 smithi194 ceph-mon[51517]: pgmap v126: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.6 MiB/s wr, 443 op/s; 148/1530 objects degraded (9.673%) 2024-09-17T00:06:14.129 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:14 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:06:14.052+0000 7f13b167f640 -1 Falling back to public interface 2024-09-17T00:06:14.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 180745 8.99 MB/sec execute 358 sec latency 53.595 ms 2024-09-17T00:06:14.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:14 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 148/1530 objects degraded (9.673%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:14 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 148/1530 objects degraded (9.673%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:15.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 181142 9.00 MB/sec execute 359 sec latency 49.592 ms 2024-09-17T00:06:15.909 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:15 smithi007 ceph-mon[65487]: pgmap v127: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.9 MiB/s wr, 356 op/s; 148/1530 objects degraded (9.673%) 2024-09-17T00:06:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:15 smithi194 ceph-mon[51517]: pgmap v127: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 2.9 MiB/s wr, 356 op/s; 148/1530 objects degraded (9.673%) 2024-09-17T00:06:16.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 181379 8.99 MB/sec execute 360 sec latency 44.805 ms 2024-09-17T00:06:17.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 181589 8.97 MB/sec execute 361 sec latency 65.922 ms 2024-09-17T00:06:17.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:17 smithi007 ceph-mon[65487]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.4 MiB/s wr, 333 op/s; 152/1674 objects degraded (9.080%) 2024-09-17T00:06:18.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:17 smithi194 ceph-mon[51517]: pgmap v128: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.4 MiB/s wr, 333 op/s; 152/1674 objects degraded (9.080%) 2024-09-17T00:06:18.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 181815 8.95 MB/sec execute 362 sec latency 58.529 ms 2024-09-17T00:06:19.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 182456 8.97 MB/sec execute 363 sec latency 35.425 ms 2024-09-17T00:06:19.732 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:19 smithi007 ceph-mon[65487]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 315 op/s; 157/1695 objects degraded (9.263%) 2024-09-17T00:06:19.732 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:19 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 157/1695 objects degraded (9.263%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:20.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:19 smithi194 ceph-mon[51517]: pgmap v129: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 3.5 MiB/s wr, 315 op/s; 157/1695 objects degraded (9.263%) 2024-09-17T00:06:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:19 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 157/1695 objects degraded (9.263%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:20.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 183188 8.98 MB/sec execute 364 sec latency 26.694 ms 2024-09-17T00:06:20.741 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:20 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:06:20.583+0000 7f13b167f640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-17T00:06:21.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 183563 8.99 MB/sec execute 365 sec latency 41.377 ms 2024-09-17T00:06:21.757 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:06:21.572+0000 7f13b167f640 -1 osd.2 94 log_to_monitors true 2024-09-17T00:06:22.008 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:06:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:06:21.872+0000 7f13a942a640 -1 osd.2 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:06:22.008 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:21 smithi007 ceph-mon[65487]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 536 op/s; 166/1740 objects degraded (9.540%) 2024-09-17T00:06:22.008 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:22.008 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:21 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:22.008 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:21 smithi007 ceph-mon[65487]: from='osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T00:06:22.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:21 smithi194 ceph-mon[51517]: pgmap v130: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.8 MiB/s wr, 536 op/s; 166/1740 objects degraded (9.540%) 2024-09-17T00:06:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:21 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:21 smithi194 ceph-mon[51517]: from='osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-17T00:06:22.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 183924 9.01 MB/sec execute 366 sec latency 86.898 ms 2024-09-17T00:06:23.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:22 smithi007 ceph-mon[65487]: from='osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T00:06:23.048 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:22 smithi007 ceph-mon[65487]: osdmap e97: 8 total, 7 up, 8 in 2024-09-17T00:06:23.048 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:22 smithi007 ceph-mon[65487]: from='osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:06:23.048 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:22 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:23.048 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:22 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:23.048 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:22 smithi007 ceph-mon[65487]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 526 op/s; 161/1734 objects degraded (9.285%) 2024-09-17T00:06:23.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:22 smithi194 ceph-mon[51517]: from='osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-17T00:06:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:22 smithi194 ceph-mon[51517]: osdmap e97: 8 total, 7 up, 8 in 2024-09-17T00:06:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:22 smithi194 ceph-mon[51517]: from='osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:06:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:22 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:22 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:22 smithi194 ceph-mon[51517]: pgmap v132: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.4 MiB/s wr, 526 op/s; 161/1734 objects degraded (9.285%) 2024-09-17T00:06:23.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 184136 9.01 MB/sec execute 367 sec latency 460.320 ms 2024-09-17T00:06:24.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:23 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:06:24.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:23 smithi007 ceph-mon[65487]: osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292] boot 2024-09-17T00:06:24.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:23 smithi007 ceph-mon[65487]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T00:06:24.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:06:24.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:23 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:06:24.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:23 smithi194 ceph-mon[51517]: osd.2 [v2:172.21.15.7:6818/686470292,v1:172.21.15.7:6819/686470292] boot 2024-09-17T00:06:24.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:23 smithi194 ceph-mon[51517]: osdmap e98: 8 total, 8 up, 8 in 2024-09-17T00:06:24.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-17T00:06:24.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 184136 8.99 MB/sec execute 368 sec latency 1460.360 ms 2024-09-17T00:06:25.090 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:24 smithi007 ceph-mon[65487]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T00:06:25.090 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:24 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 161/1734 objects degraded (9.285%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:25.091 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:25.091 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:25.091 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:24 smithi007 ceph-mon[65487]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.6 MiB/s wr, 552 op/s; 161/1734 objects degraded (9.285%) 2024-09-17T00:06:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:24 smithi194 ceph-mon[51517]: osdmap e99: 8 total, 8 up, 8 in 2024-09-17T00:06:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:24 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 161/1734 objects degraded (9.285%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:24 smithi194 ceph-mon[51517]: pgmap v135: 97 pgs: 7 active+undersized, 19 active+undersized+degraded, 71 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 4.6 MiB/s wr, 552 op/s; 161/1734 objects degraded (9.285%) 2024-09-17T00:06:25.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 184239 8.97 MB/sec execute 369 sec latency 2173.913 ms 2024-09-17T00:06:25.891 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:25 smithi007 ceph-mon[65487]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T00:06:26.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:25 smithi194 ceph-mon[51517]: osdmap e100: 8 total, 8 up, 8 in 2024-09-17T00:06:26.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 184443 8.97 MB/sec execute 370 sec latency 434.697 ms 2024-09-17T00:06:27.150 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:26 smithi007 ceph-mon[65487]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T00:06:27.150 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:26 smithi007 ceph-mon[65487]: pgmap v138: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+undersized, 6 active+undersized+degraded, 84 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 107 op/s; 89/1623 objects degraded (5.484%); 1023 KiB/s, 22 objects/s recovering 2024-09-17T00:06:27.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:26 smithi194 ceph-mon[51517]: osdmap e101: 8 total, 8 up, 8 in 2024-09-17T00:06:27.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:26 smithi194 ceph-mon[51517]: pgmap v138: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 3 active+undersized, 6 active+undersized+degraded, 84 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 107 op/s; 89/1623 objects degraded (5.484%); 1023 KiB/s, 22 objects/s recovering 2024-09-17T00:06:27.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 184490 8.95 MB/sec execute 371 sec latency 1217.035 ms 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:27.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:06:27.908 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:27.908 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:27.908 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:27.908 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:27.908 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:27.908 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:27 smithi007 ceph-mon[65487]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: osdmap e102: 8 total, 8 up, 8 in 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:28.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:06:28.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:28.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:28.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:28.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:28.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:28.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:27 smithi194 ceph-mon[51517]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-17T00:06:28.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 184860 8.95 MB/sec execute 372 sec latency 83.488 ms 2024-09-17T00:06:28.927 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:28 smithi007 ceph-mon[65487]: pgmap v140: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 92 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 288 op/s; 47/1626 objects degraded (2.891%); 1.4 MiB/s, 23 objects/s recovering 2024-09-17T00:06:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:28 smithi194 ceph-mon[51517]: pgmap v140: 97 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 92 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.8 MiB/s wr, 288 op/s; 47/1626 objects degraded (2.891%); 1.4 MiB/s, 23 objects/s recovering 2024-09-17T00:06:29.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 185078 8.94 MB/sec execute 373 sec latency 50.794 ms 2024-09-17T00:06:29.955 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:29 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 47/1626 objects degraded (2.891%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:30.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:29 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 47/1626 objects degraded (2.891%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:30.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 185385 8.92 MB/sec execute 374 sec latency 61.235 ms 2024-09-17T00:06:30.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:30 smithi007 ceph-mon[65487]: pgmap v141: 97 pgs: 1 peering, 1 active+undersized+remapped, 95 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 275 op/s; 1.5 MiB/s, 26 objects/s recovering 2024-09-17T00:06:31.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:30 smithi194 ceph-mon[51517]: pgmap v141: 97 pgs: 1 peering, 1 active+undersized+remapped, 95 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.0 MiB/s wr, 275 op/s; 1.5 MiB/s, 26 objects/s recovering 2024-09-17T00:06:31.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 185675 8.92 MB/sec execute 375 sec latency 114.779 ms 2024-09-17T00:06:32.007 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:31 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/1626 objects degraded (2.891%), 2 pgs degraded) 2024-09-17T00:06:32.007 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:31 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:06:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:31 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/1626 objects degraded (2.891%), 2 pgs degraded) 2024-09-17T00:06:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:31 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:06:32.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 186031 8.91 MB/sec execute 376 sec latency 90.790 ms 2024-09-17T00:06:32.571 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:06:33.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:32 smithi007 ceph-mon[65487]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:33.020 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:32 smithi007 ceph-mon[65487]: pgmap v142: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 407 op/s; 1.2 MiB/s, 20 objects/s recovering 2024-09-17T00:06:33.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:32 smithi194 ceph-mon[51517]: from='client.34282 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:33.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:32 smithi194 ceph-mon[51517]: pgmap v142: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.4 KiB/s rd, 4.6 MiB/s wr, 407 op/s; 1.2 MiB/s, 20 objects/s recovering 2024-09-17T00:06:33.299 INFO:teuthology.orchestra.run.smithi007.stdout: 5 186394 8.92 MB/sec execute 377 sec latency 55.484 ms 2024-09-17T00:06:33.572 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:06:33.572 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (5m) 12s ago 14m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (2m) 12s ago 14m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (2m) 2m ago 14m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (5m) 12s ago 14m 54.0M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (8m) 12s ago 8m 37.3M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (8m) 2m ago 8m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (4m) 12s ago 16m 502M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (4m) 2m ago 14m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (3m) 12s ago 16m 51.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (3m) 2m ago 14m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (8m) 12s ago 8m 256M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (8m) 2m ago 8m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:06:33.573 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (5m) 12s ago 14m 16.0M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (5m) 2m ago 13m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (116s) 12s ago 13m 273M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (69s) 12s ago 12m 212M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (21s) 12s ago 12m 12.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (11m) 12s ago 11m 553M 1937M 17.2.0 e1d6a67b021e a7d457ae734f 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (11m) 2m ago 11m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (11m) 2m ago 11m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (10m) 2m ago 10m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (10m) 2m ago 10m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:06:33.574 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (4m) 12s ago 13m 64.1M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:06:34.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:33 smithi007 ceph-mon[65487]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:34.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:33 smithi007 ceph-mon[65487]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:34.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:33 smithi194 ceph-mon[51517]: from='client.34286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:33 smithi194 ceph-mon[51517]: from='client.34290 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:34.212 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:06:34.212 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:06:34.212 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:06:34.213 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:06:34.214 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-17T00:06:34.214 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T00:06:34.214 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:06:34.214 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:06:34.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 186775 8.90 MB/sec execute 378 sec latency 56.561 ms 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:06:34.752 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:06:34.753 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "9/23 daemons upgraded", 2024-09-17T00:06:34.753 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:06:34.753 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:06:34.753 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:06:35.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:34 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3807940489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:35.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:34 smithi007 ceph-mon[65487]: pgmap v143: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.8 MiB/s wr, 312 op/s; 602 KiB/s, 8 objects/s recovering 2024-09-17T00:06:35.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:34 smithi007 ceph-mon[65487]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:35.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 187123 8.91 MB/sec execute 379 sec latency 42.421 ms 2024-09-17T00:06:35.360 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:06:35.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:34 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3807940489' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:06:35.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:34 smithi194 ceph-mon[51517]: pgmap v143: 97 pgs: 97 active+clean; 343 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 2.8 MiB/s wr, 312 op/s; 602 KiB/s, 8 objects/s recovering 2024-09-17T00:06:35.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:34 smithi194 ceph-mon[51517]: from='client.34298 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:06:36.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:35 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/4158207580' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:06:36.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 187589 8.94 MB/sec execute 380 sec latency 53.982 ms 2024-09-17T00:06:36.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:35 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/4158207580' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:06:37.079 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:36 smithi007 ceph-mon[65487]: pgmap v144: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 512 op/s; 531 KiB/s, 7 objects/s recovering 2024-09-17T00:06:37.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 187936 8.95 MB/sec execute 381 sec latency 51.370 ms 2024-09-17T00:06:37.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:36 smithi194 ceph-mon[51517]: pgmap v144: 97 pgs: 97 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 512 op/s; 531 KiB/s, 7 objects/s recovering 2024-09-17T00:06:38.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 188161 8.95 MB/sec execute 382 sec latency 76.108 ms 2024-09-17T00:06:39.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 188328 8.94 MB/sec execute 383 sec latency 78.677 ms 2024-09-17T00:06:39.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:39 smithi007 ceph-mon[65487]: pgmap v145: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 371 op/s; 211 KiB/s, 4 objects/s recovering 2024-09-17T00:06:39.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:39.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:39.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:40.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:39 smithi194 ceph-mon[51517]: pgmap v145: 97 pgs: 97 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 371 op/s; 211 KiB/s, 4 objects/s recovering 2024-09-17T00:06:40.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:40.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:40.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:40.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 188519 8.93 MB/sec execute 384 sec latency 67.302 ms 2024-09-17T00:06:41.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 188701 8.91 MB/sec execute 385 sec latency 83.168 ms 2024-09-17T00:06:41.992 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:41 smithi007 ceph-mon[65487]: pgmap v146: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 505 op/s; 208 KiB/s, 3 objects/s recovering 2024-09-17T00:06:42.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:41 smithi194 ceph-mon[51517]: pgmap v146: 97 pgs: 97 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.9 MiB/s wr, 505 op/s; 208 KiB/s, 3 objects/s recovering 2024-09-17T00:06:42.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 188967 8.89 MB/sec execute 386 sec latency 44.734 ms 2024-09-17T00:06:43.018 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:43.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:43.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 189233 8.89 MB/sec execute 387 sec latency 61.962 ms 2024-09-17T00:06:44.031 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:43 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:44.031 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:43 smithi007 ceph-mon[65487]: Upgrade: osd.3 is safe to restart 2024-09-17T00:06:44.031 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:43 smithi007 ceph-mon[65487]: pgmap v147: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 273 B/s, 0 objects/s recovering 2024-09-17T00:06:44.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:43 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-17T00:06:44.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:43 smithi194 ceph-mon[51517]: Upgrade: osd.3 is safe to restart 2024-09-17T00:06:44.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:43 smithi194 ceph-mon[51517]: pgmap v147: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.0 MiB/s wr, 517 op/s; 273 B/s, 0 objects/s recovering 2024-09-17T00:06:44.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 189553 8.89 MB/sec execute 388 sec latency 71.660 ms 2024-09-17T00:06:45.296 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:45 smithi007 ceph-mon[65487]: Upgrade: Updating osd.3 2024-09-17T00:06:45.296 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:45.296 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T00:06:45.296 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:06:45.296 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:45 smithi007 ceph-mon[65487]: Deploying daemon osd.3 on smithi007 2024-09-17T00:06:45.296 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:45 smithi007 ceph-mon[65487]: pgmap v148: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-09-17T00:06:45.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 189882 8.89 MB/sec execute 389 sec latency 57.584 ms 2024-09-17T00:06:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:45 smithi194 ceph-mon[51517]: Upgrade: Updating osd.3 2024-09-17T00:06:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-17T00:06:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:06:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:45 smithi194 ceph-mon[51517]: Deploying daemon osd.3 on smithi007 2024-09-17T00:06:45.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:45 smithi194 ceph-mon[51517]: pgmap v148: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 389 op/s 2024-09-17T00:06:46.300 INFO:teuthology.orchestra.run.smithi007.stdout: 5 190430 8.88 MB/sec execute 390 sec latency 39.084 ms 2024-09-17T00:06:47.303 INFO:teuthology.orchestra.run.smithi007.stdout: 5 190826 8.90 MB/sec execute 391 sec latency 34.861 ms 2024-09-17T00:06:47.981 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:47 smithi007 ceph-mon[65487]: pgmap v149: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 451 op/s 2024-09-17T00:06:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:47 smithi194 ceph-mon[51517]: pgmap v149: 97 pgs: 97 active+clean; 332 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 451 op/s 2024-09-17T00:06:48.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 191312 8.93 MB/sec execute 392 sec latency 32.618 ms 2024-09-17T00:06:49.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 191634 8.94 MB/sec execute 393 sec latency 54.479 ms 2024-09-17T00:06:49.513 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:49 smithi007 systemd[1]: Stopping Ceph osd.3 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:06:49.765 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:49 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[45093]: 2024-09-17T00:06:49.734+0000 7f07efe3b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:06:49.765 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:49 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[45093]: 2024-09-17T00:06:49.734+0000 7f07efe3b700 -1 osd.3 102 *** Got signal Terminated *** 2024-09-17T00:06:49.766 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:49 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[45093]: 2024-09-17T00:06:49.734+0000 7f07efe3b700 -1 osd.3 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:06:50.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:49 smithi007 ceph-mon[65487]: pgmap v150: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 406 op/s 2024-09-17T00:06:50.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:49 smithi007 ceph-mon[65487]: osd.3 marked itself down and dead 2024-09-17T00:06:50.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:49 smithi194 ceph-mon[51517]: pgmap v150: 97 pgs: 97 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 406 op/s 2024-09-17T00:06:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:49 smithi194 ceph-mon[51517]: osd.3 marked itself down and dead 2024-09-17T00:06:50.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 191924 8.93 MB/sec execute 394 sec latency 476.320 ms 2024-09-17T00:06:50.551 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:50 smithi007 podman[78923]: 2024-09-17 00:06:50.531262112 +0000 UTC m=+0.920376847 container died a7d457ae734f00d3e98040d9a164c3cec4d1b3e897bbaf22edc4d70b8d8defdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, release=754, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8) 2024-09-17T00:06:51.064 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:50 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:06:51.065 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:50 smithi007 ceph-mon[65487]: osdmap e103: 8 total, 7 up, 8 in 2024-09-17T00:06:51.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:50 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:06:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:50 smithi194 ceph-mon[51517]: osdmap e103: 8 total, 7 up, 8 in 2024-09-17T00:06:51.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 192064 8.92 MB/sec execute 395 sec latency 991.632 ms 2024-09-17T00:06:51.580 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:51 smithi007 podman[78923]: 2024-09-17 00:06:51.332061823 +0000 UTC m=+1.721176560 container cleanup a7d457ae734f00d3e98040d9a164c3cec4d1b3e897bbaf22edc4d70b8d8defdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, name=centos-stream, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, version=8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-17T00:06:51.580 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:51 smithi007 bash[78923]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3 2024-09-17T00:06:51.831 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:51 smithi007 ceph-mon[65487]: pgmap v152: 97 pgs: 21 peering, 10 stale+active+clean, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 338 op/s 2024-09-17T00:06:51.831 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:51 smithi007 ceph-mon[65487]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T00:06:51.831 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:51 smithi007 ceph-mon[65487]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T00:06:52.093 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:51 smithi007 podman[78940]: 2024-09-17 00:06:51.907885732 +0000 UTC m=+1.374205897 container remove a7d457ae734f00d3e98040d9a164c3cec4d1b3e897bbaf22edc4d70b8d8defdc (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, vendor=Red Hat, Inc., version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, io.openshift.expose-services=, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, name=centos-stream, ceph=True) 2024-09-17T00:06:52.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:51 smithi194 ceph-mon[51517]: pgmap v152: 97 pgs: 21 peering, 10 stale+active+clean, 66 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.5 MiB/s wr, 338 op/s 2024-09-17T00:06:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:51 smithi194 ceph-mon[51517]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-17T00:06:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:51 smithi194 ceph-mon[51517]: osdmap e104: 8 total, 7 up, 8 in 2024-09-17T00:06:52.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 192293 8.91 MB/sec execute 396 sec latency 41.896 ms 2024-09-17T00:06:52.561 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:52 smithi007 podman[79025]: 2024-09-17 00:06:52.310847543 +0000 UTC m=+0.020052395 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:06:52.562 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:52 smithi007 podman[79025]: 2024-09-17 00:06:52.465842935 +0000 UTC m=+0.175047768 container create 8946daae0d393687af1b6d3bb94dbc991caa67a2d6d04393bac99481efb315d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T00:06:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:52 smithi007 ceph-mon[65487]: pgmap v154: 97 pgs: 4 active+undersized, 27 peering, 10 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.8 MiB/s wr, 583 op/s; 83/1698 objects degraded (4.888%) 2024-09-17T00:06:52.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:52 smithi007 podman[79025]: 2024-09-17 00:06:52.836189971 +0000 UTC m=+0.545394809 container init 8946daae0d393687af1b6d3bb94dbc991caa67a2d6d04393bac99481efb315d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T00:06:52.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:52 smithi007 podman[79025]: 2024-09-17 00:06:52.844260871 +0000 UTC m=+0.553465699 container start 8946daae0d393687af1b6d3bb94dbc991caa67a2d6d04393bac99481efb315d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T00:06:53.104 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:52 smithi007 podman[79025]: 2024-09-17 00:06:52.986335779 +0000 UTC m=+0.695540614 container attach 8946daae0d393687af1b6d3bb94dbc991caa67a2d6d04393bac99481efb315d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-17T00:06:53.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:52 smithi194 ceph-mon[51517]: pgmap v154: 97 pgs: 4 active+undersized, 27 peering, 10 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 5.8 MiB/s wr, 583 op/s; 83/1698 objects degraded (4.888%) 2024-09-17T00:06:53.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 192656 8.91 MB/sec execute 397 sec latency 47.956 ms 2024-09-17T00:06:53.362 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:53 smithi007 podman[79025]: 2024-09-17 00:06:53.187876777 +0000 UTC m=+0.897081614 container died 8946daae0d393687af1b6d3bb94dbc991caa67a2d6d04393bac99481efb315d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T00:06:53.885 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:53 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 83/1698 objects degraded (4.888%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:54.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:53 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 83/1698 objects degraded (4.888%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T00:06:54.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 193092 8.90 MB/sec execute 398 sec latency 55.661 ms 2024-09-17T00:06:54.389 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:54 smithi007 podman[79047]: 2024-09-17 00:06:54.327232551 +0000 UTC m=+1.131503314 container remove 8946daae0d393687af1b6d3bb94dbc991caa67a2d6d04393bac99481efb315d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97) 2024-09-17T00:06:54.389 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:54 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-17T00:06:54.389 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:54 smithi007 systemd[1]: Stopped Ceph osd.3 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:06:54.389 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:54 smithi007 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.3.service: Consumed 3min 38.815s CPU time. 2024-09-17T00:06:54.892 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:54 smithi007 systemd[1]: Starting Ceph osd.3 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:06:55.146 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 podman[79138]: 2024-09-17 00:06:55.013808832 +0000 UTC m=+0.028797493 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:06:55.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 193510 8.90 MB/sec execute 399 sec latency 33.385 ms 2024-09-17T00:06:55.399 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 podman[79138]: 2024-09-17 00:06:55.177369179 +0000 UTC m=+0.192357824 container create 57a6fe659e6fda50870893b7dbe342301b7fc977451d06924f15feb199624cf3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T00:06:55.650 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 podman[79138]: 2024-09-17 00:06:55.533071363 +0000 UTC m=+0.548060020 container init 57a6fe659e6fda50870893b7dbe342301b7fc977451d06924f15feb199624cf3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T00:06:55.650 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 podman[79138]: 2024-09-17 00:06:55.542279611 +0000 UTC m=+0.557268271 container start 57a6fe659e6fda50870893b7dbe342301b7fc977451d06924f15feb199624cf3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T00:06:55.650 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 podman[79138]: 2024-09-17 00:06:55.641571903 +0000 UTC m=+0.656560552 container attach 57a6fe659e6fda50870893b7dbe342301b7fc977451d06924f15feb199624cf3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-17T00:06:55.905 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:55.905 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 bash[79138]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:55.905 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:55.906 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:55 smithi007 bash[79138]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:06:55.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:55.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:55.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:55 smithi007 ceph-mon[65487]: pgmap v155: 97 pgs: 4 active+undersized, 27 peering, 10 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 83/1698 objects degraded (4.888%) 2024-09-17T00:06:56.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:06:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:55 smithi194 ceph-mon[51517]: pgmap v155: 97 pgs: 4 active+undersized, 27 peering, 10 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.9 MiB/s wr, 490 op/s; 83/1698 objects degraded (4.888%) 2024-09-17T00:06:56.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 193973 8.91 MB/sec execute 400 sec latency 32.189 ms 2024-09-17T00:06:56.411 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T00:06:56.411 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T00:06:56.411 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T00:06:56.411 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T00:06:56.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-mon[65487]: pgmap v156: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s wr, 566 op/s; 199/1641 objects degraded (12.127%) 2024-09-17T00:06:56.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-mon[65487]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate[79152]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 bash[79138]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-17T00:06:56.923 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:56 smithi007 podman[79138]: 2024-09-17 00:06:56.787013078 +0000 UTC m=+1.802001737 container died 57a6fe659e6fda50870893b7dbe342301b7fc977451d06924f15feb199624cf3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:06:57.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:56 smithi194 ceph-mon[51517]: pgmap v156: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 MiB/s wr, 566 op/s; 199/1641 objects degraded (12.127%) 2024-09-17T00:06:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:56 smithi194 ceph-mon[51517]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-09-17T00:06:57.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 194417 8.92 MB/sec execute 401 sec latency 37.074 ms 2024-09-17T00:06:57.959 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:57 smithi007 podman[79295]: 2024-09-17 00:06:57.779077169 +0000 UTC m=+0.982833194 container remove 57a6fe659e6fda50870893b7dbe342301b7fc977451d06924f15feb199624cf3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:06:58.215 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 podman[79351]: 2024-09-17 00:06:57.916078884 +0000 UTC m=+0.025518723 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:06:58.215 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 podman[79351]: 2024-09-17 00:06:58.100114454 +0000 UTC m=+0.209554275 container create 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-17T00:06:58.308 INFO:teuthology.orchestra.run.smithi007.stdout: 5 194826 8.94 MB/sec execute 402 sec latency 43.477 ms 2024-09-17T00:06:58.472 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 podman[79351]: 2024-09-17 00:06:58.470523723 +0000 UTC m=+0.579963553 container init 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-17T00:06:58.724 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 podman[79351]: 2024-09-17 00:06:58.478554844 +0000 UTC m=+0.587994668 container start 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:06:58.724 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 ceph-osd[79368]: -- 172.21.15.7:0/3503896363 <== mon.0 v2:172.21.15.7:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5653198fa1e0 con 0x5653198d0000 2024-09-17T00:06:58.724 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 bash[79351]: 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 2024-09-17T00:06:58.724 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:06:58 smithi007 systemd[1]: Started Ceph osd.3 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:06:59.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 195233 8.95 MB/sec execute 403 sec latency 46.565 ms 2024-09-17T00:06:59.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:59 smithi007 ceph-mon[65487]: pgmap v157: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.9 MiB/s wr, 537 op/s; 197/1614 objects degraded (12.206%) 2024-09-17T00:06:59.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:59.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:06:59.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:59 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:06:59.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:06:59 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 197/1614 objects degraded (12.206%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:59 smithi194 ceph-mon[51517]: pgmap v157: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 341 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.9 MiB/s wr, 537 op/s; 197/1614 objects degraded (12.206%) 2024-09-17T00:07:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:59 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:59 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:59 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:00.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:06:59 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 197/1614 objects degraded (12.206%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:00.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 195621 8.96 MB/sec execute 404 sec latency 41.748 ms 2024-09-17T00:07:01.009 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:07:00 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:07:00.863+0000 7f2383437640 -1 Falling back to public interface 2024-09-17T00:07:01.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 195833 8.94 MB/sec execute 405 sec latency 61.555 ms 2024-09-17T00:07:01.795 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:01 smithi007 ceph-mon[65487]: pgmap v158: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 218/1671 objects degraded (13.046%) 2024-09-17T00:07:02.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:01 smithi194 ceph-mon[51517]: pgmap v158: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.7 MiB/s wr, 573 op/s; 218/1671 objects degraded (13.046%) 2024-09-17T00:07:02.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 196136 8.94 MB/sec execute 406 sec latency 45.156 ms 2024-09-17T00:07:03.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 196429 8.93 MB/sec execute 407 sec latency 66.965 ms 2024-09-17T00:07:03.816 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:03 smithi007 ceph-mon[65487]: pgmap v159: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.7 MiB/s wr, 566 op/s; 221/1701 objects degraded (12.992%) 2024-09-17T00:07:04.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:03 smithi194 ceph-mon[51517]: pgmap v159: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.4 KiB/s rd, 5.7 MiB/s wr, 566 op/s; 221/1701 objects degraded (12.992%) 2024-09-17T00:07:04.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 196872 8.92 MB/sec execute 408 sec latency 46.282 ms 2024-09-17T00:07:04.827 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:04 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 221/1701 objects degraded (12.992%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:05.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:04 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 221/1701 objects degraded (12.992%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:05.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 197201 8.93 MB/sec execute 409 sec latency 33.377 ms 2024-09-17T00:07:05.839 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:05 smithi007 ceph-mon[65487]: pgmap v160: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 497 op/s; 221/1701 objects degraded (12.992%) 2024-09-17T00:07:05.995 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:07:06.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:05 smithi194 ceph-mon[51517]: pgmap v160: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 497 op/s; 221/1701 objects degraded (12.992%) 2024-09-17T00:07:06.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 197540 8.93 MB/sec execute 410 sec latency 58.121 ms 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (6m) 46s ago 15m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (3m) 46s ago 15m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (3m) 2m ago 14m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (6m) 46s ago 15m 54.0M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (9m) 46s ago 9m 37.3M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:07:07.138 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (9m) 2m ago 9m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (5m) 46s ago 16m 502M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (4m) 2m ago 14m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (4m) 46s ago 16m 51.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (3m) 2m ago 14m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (9m) 46s ago 9m 256M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (8m) 2m ago 8m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (6m) 46s ago 15m 16.0M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (5m) 2m ago 14m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (2m) 46s ago 13m 273M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (103s) 46s ago 13m 212M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (55s) 46s ago 12m 12.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 starting - - - 1937M 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (12m) 2m ago 12m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (11m) 2m ago 11m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:07:07.139 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (11m) 2m ago 11m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:07:07.140 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (10m) 2m ago 10m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:07:07.140 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (4m) 46s ago 14m 64.1M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:07:07.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 197925 8.93 MB/sec execute 411 sec latency 72.375 ms 2024-09-17T00:07:07.876 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:07 smithi007 ceph-mon[65487]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:07.876 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:07 smithi007 ceph-mon[65487]: from='client.44229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:07.876 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:07 smithi007 ceph-mon[65487]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 620 op/s; 197/1587 objects degraded (12.413%) 2024-09-17T00:07:07.876 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:07 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:07.876 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:07 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:07.893 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:07:07.893 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:07:07.893 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:07:07.893 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:07.893 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 3 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:07.894 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:07:07.895 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-17T00:07:07.895 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T00:07:07.895 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:07:07.895 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:07:08.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:07 smithi194 ceph-mon[51517]: from='client.44227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:07 smithi194 ceph-mon[51517]: from='client.44229 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:07 smithi194 ceph-mon[51517]: pgmap v161: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 338 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.9 MiB/s wr, 620 op/s; 197/1587 objects degraded (12.413%) 2024-09-17T00:07:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:07 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:07 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:08.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 198203 8.94 MB/sec execute 412 sec latency 47.580 ms 2024-09-17T00:07:08.485 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "10/23 daemons upgraded", 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:07:08.486 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:07:08.641 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:07:08 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:07:08.483+0000 7f2383437640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-17T00:07:08.892 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:08 smithi007 ceph-mon[65487]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:08.892 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:08 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3008834134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:08.892 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:08.892 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:08 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:09.106 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 197/1587 objects degraded (12.413%), 27 pgs degraded 2024-09-17T00:07:09.106 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: osd.3 (root=default,host=smithi007) is down 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 197/1587 objects degraded (12.413%), 27 pgs degraded 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.1 is active+undersized+degraded, acting [2,0] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.9 is active+undersized+degraded, acting [1,7] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.a is active+undersized+degraded, acting [1,7] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.d is active+undersized+degraded, acting [1,4] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.e is active+undersized+degraded, acting [2,7] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.12 is active+undersized+degraded, acting [5,7] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.13 is active+undersized+degraded, acting [0,4] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.14 is active+undersized+degraded, acting [6,5] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.19 is active+undersized+degraded, acting [6,0] 2024-09-17T00:07:09.107 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.1b is active+undersized+degraded, acting [7,6] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.1e is active+undersized+degraded, acting [0,5] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1 is active+undersized+degraded, acting [0,4] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.2 is active+undersized+degraded, acting [5,6] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.5 is active+undersized+degraded, acting [5,2] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.7 is active+undersized+degraded, acting [7,0] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.8 is active+undersized+degraded, acting [1,7] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.b is active+undersized+degraded, acting [0,4] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.c is active+undersized+degraded, acting [5,6] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.12 is active+undersized+degraded, acting [0,7] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.15 is active+undersized+degraded, acting [7,4] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.17 is active+undersized+degraded, acting [0,5] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.18 is active+undersized+degraded, acting [0,1] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.19 is active+undersized+degraded, acting [1,4] 2024-09-17T00:07:09.108 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1e is active+undersized+degraded, acting [6,2] 2024-09-17T00:07:09.109 INFO:teuthology.orchestra.run.smithi007.stdout: pg 4.0 is active+undersized+degraded, acting [7,0] 2024-09-17T00:07:09.109 INFO:teuthology.orchestra.run.smithi007.stdout: pg 4.14 is active+undersized+degraded, acting [1,7] 2024-09-17T00:07:09.109 INFO:teuthology.orchestra.run.smithi007.stdout: pg 4.19 is active+undersized+degraded, acting [2,0] 2024-09-17T00:07:09.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:08 smithi194 ceph-mon[51517]: from='client.34316 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:09.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:08 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3008834134' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:09.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:08 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:09.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:08 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:09.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 198477 8.95 MB/sec execute 413 sec latency 55.411 ms 2024-09-17T00:07:09.915 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-mon[65487]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:09.915 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-mon[65487]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.0 MiB/s wr, 483 op/s; 202/1620 objects degraded (12.469%) 2024-09-17T00:07:09.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2537398672' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:07:09.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 202/1620 objects degraded (12.469%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:09.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:09.916 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:09.916 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:07:09 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:07:09.831+0000 7f2383437640 -1 osd.3 102 log_to_monitors true 2024-09-17T00:07:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:09 smithi194 ceph-mon[51517]: from='client.34324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:09 smithi194 ceph-mon[51517]: pgmap v162: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 340 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.0 MiB/s wr, 483 op/s; 202/1620 objects degraded (12.469%) 2024-09-17T00:07:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:09 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2537398672' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:07:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:09 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 202/1620 objects degraded (12.469%), 27 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:10.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 198674 8.94 MB/sec execute 414 sec latency 45.684 ms 2024-09-17T00:07:10.920 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:10 smithi007 ceph-mon[65487]: from='osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T00:07:10.920 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:07:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:07:10.826+0000 7f237b1e2640 -1 osd.3 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:07:11.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:10 smithi194 ceph-mon[51517]: from='osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-17T00:07:11.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 198961 8.93 MB/sec execute 415 sec latency 64.436 ms 2024-09-17T00:07:12.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:11 smithi007 ceph-mon[65487]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 606 op/s; 209/1668 objects degraded (12.530%) 2024-09-17T00:07:12.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:11 smithi007 ceph-mon[65487]: from='osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T00:07:12.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:11 smithi007 ceph-mon[65487]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T00:07:12.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:11 smithi007 ceph-mon[65487]: from='osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:07:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:11 smithi194 ceph-mon[51517]: pgmap v163: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 343 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 6.2 MiB/s wr, 606 op/s; 209/1668 objects degraded (12.530%) 2024-09-17T00:07:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:11 smithi194 ceph-mon[51517]: from='osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-17T00:07:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:11 smithi194 ceph-mon[51517]: osdmap e105: 8 total, 7 up, 8 in 2024-09-17T00:07:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:11 smithi194 ceph-mon[51517]: from='osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi007", "root=default"]}]: dispatch 2024-09-17T00:07:12.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 199148 8.92 MB/sec execute 416 sec latency 549.269 ms 2024-09-17T00:07:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:12 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:07:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:12 smithi007 ceph-mon[65487]: osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483] boot 2024-09-17T00:07:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:12 smithi007 ceph-mon[65487]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T00:07:13.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:12 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:07:13.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:12 smithi007 ceph-mon[65487]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T00:07:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:12 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:07:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:12 smithi194 ceph-mon[51517]: osd.3 [v2:172.21.15.7:6826/779511483,v1:172.21.15.7:6827/779511483] boot 2024-09-17T00:07:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:12 smithi194 ceph-mon[51517]: osdmap e106: 8 total, 8 up, 8 in 2024-09-17T00:07:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:12 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-17T00:07:13.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:12 smithi194 ceph-mon[51517]: osdmap e107: 8 total, 8 up, 8 in 2024-09-17T00:07:13.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 199148 8.90 MB/sec execute 417 sec latency 1549.297 ms 2024-09-17T00:07:14.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: pgmap v166: 97 pgs: 1 unknown, 8 peering, 11 active+undersized, 21 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 154/1683 objects degraded (9.150%) 2024-09-17T00:07:14.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.039 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:14.040 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:13 smithi007 ceph-mon[65487]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: pgmap v166: 97 pgs: 1 unknown, 8 peering, 11 active+undersized, 21 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.3 MiB/s wr, 520 op/s; 154/1683 objects degraded (9.150%) 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:14.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:14.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:14.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:13 smithi194 ceph-mon[51517]: osdmap e108: 8 total, 8 up, 8 in 2024-09-17T00:07:14.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 199253 8.89 MB/sec execute 418 sec latency 2116.911 ms 2024-09-17T00:07:14.806 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:14 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:14.806 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:14 smithi007 ceph-mon[65487]: Upgrade: 1 pgs have unknown state; cannot draw any conclusions 2024-09-17T00:07:14.806 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:14 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 154/1683 objects degraded (9.150%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:14.806 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:14 smithi007 ceph-mon[65487]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T00:07:15.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:14 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:14 smithi194 ceph-mon[51517]: Upgrade: 1 pgs have unknown state; cannot draw any conclusions 2024-09-17T00:07:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:14 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 154/1683 objects degraded (9.150%), 21 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:14 smithi194 ceph-mon[51517]: osdmap e109: 8 total, 8 up, 8 in 2024-09-17T00:07:15.309 INFO:teuthology.orchestra.run.smithi007.stdout: 5 199392 8.87 MB/sec execute 419 sec latency 258.306 ms 2024-09-17T00:07:15.839 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:15 smithi007 ceph-mon[65487]: pgmap v170: 97 pgs: 1 unknown, 8 peering, 11 active+undersized, 21 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.1 MiB/s wr, 33 op/s; 154/1683 objects degraded (9.150%) 2024-09-17T00:07:15.839 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:15 smithi007 ceph-mon[65487]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T00:07:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:15 smithi194 ceph-mon[51517]: pgmap v170: 97 pgs: 1 unknown, 8 peering, 11 active+undersized, 21 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.1 MiB/s wr, 33 op/s; 154/1683 objects degraded (9.150%) 2024-09-17T00:07:16.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:15 smithi194 ceph-mon[51517]: osdmap e110: 8 total, 8 up, 8 in 2024-09-17T00:07:16.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 199622 8.86 MB/sec execute 420 sec latency 630.800 ms 2024-09-17T00:07:17.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 200005 8.86 MB/sec execute 421 sec latency 965.315 ms 2024-09-17T00:07:17.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:17 smithi007 ceph-mon[65487]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T00:07:17.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:17 smithi007 ceph-mon[65487]: pgmap v173: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 9 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 81 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s wr, 273 op/s; 110/1560 objects degraded (7.051%); 6.1 KiB/s, 27 objects/s recovering 2024-09-17T00:07:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:17 smithi194 ceph-mon[51517]: osdmap e111: 8 total, 8 up, 8 in 2024-09-17T00:07:18.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:17 smithi194 ceph-mon[51517]: pgmap v173: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 unknown, 9 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+degraded, 81 active+clean; 340 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.3 MiB/s wr, 273 op/s; 110/1560 objects degraded (7.051%); 6.1 KiB/s, 27 objects/s recovering 2024-09-17T00:07:18.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 200424 8.86 MB/sec execute 422 sec latency 565.163 ms 2024-09-17T00:07:18.937 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:18 smithi007 ceph-mon[65487]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T00:07:19.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:18 smithi194 ceph-mon[51517]: osdmap e112: 8 total, 8 up, 8 in 2024-09-17T00:07:19.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 200619 8.85 MB/sec execute 423 sec latency 948.736 ms 2024-09-17T00:07:19.704 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:19 smithi007 ceph-mon[65487]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T00:07:19.704 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:19 smithi007 ceph-mon[65487]: pgmap v176: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.9 MiB/s wr, 303 op/s; 190/1512 objects degraded (12.566%); 1.1 MiB/s, 33 objects/s recovering 2024-09-17T00:07:19.704 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:19 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 110/1560 objects degraded (7.051%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:19.704 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:19 smithi007 ceph-mon[65487]: osdmap e114: 8 total, 8 up, 8 in 2024-09-17T00:07:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:19 smithi194 ceph-mon[51517]: osdmap e113: 8 total, 8 up, 8 in 2024-09-17T00:07:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:19 smithi194 ceph-mon[51517]: pgmap v176: 97 pgs: 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 87 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.9 MiB/s wr, 303 op/s; 190/1512 objects degraded (12.566%); 1.1 MiB/s, 33 objects/s recovering 2024-09-17T00:07:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:19 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 110/1560 objects degraded (7.051%), 6 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:19 smithi194 ceph-mon[51517]: osdmap e114: 8 total, 8 up, 8 in 2024-09-17T00:07:20.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 200800 8.84 MB/sec execute 424 sec latency 915.113 ms 2024-09-17T00:07:21.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 201024 8.83 MB/sec execute 425 sec latency 959.476 ms 2024-09-17T00:07:21.701 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:21 smithi007 ceph-mon[65487]: osdmap e115: 8 total, 8 up, 8 in 2024-09-17T00:07:21.701 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:21 smithi007 ceph-mon[65487]: pgmap v179: 97 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 453 op/s; 138/1500 objects degraded (9.200%); 1.2 MiB/s, 23 objects/s recovering 2024-09-17T00:07:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:21 smithi194 ceph-mon[51517]: osdmap e115: 8 total, 8 up, 8 in 2024-09-17T00:07:22.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:21 smithi194 ceph-mon[51517]: pgmap v179: 97 pgs: 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 339 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 453 op/s; 138/1500 objects degraded (9.200%); 1.2 MiB/s, 23 objects/s recovering 2024-09-17T00:07:22.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 201474 8.84 MB/sec execute 426 sec latency 55.583 ms 2024-09-17T00:07:22.878 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:22 smithi007 ceph-mon[65487]: pgmap v180: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 2.9 MiB/s wr, 397 op/s; 412 KiB/s, 40 objects/s recovering 2024-09-17T00:07:22.878 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:22 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 138/1500 objects degraded (9.200%), 5 pgs degraded) 2024-09-17T00:07:22.878 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:22 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:07:23.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:22 smithi194 ceph-mon[51517]: pgmap v180: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.1 KiB/s rd, 2.9 MiB/s wr, 397 op/s; 412 KiB/s, 40 objects/s recovering 2024-09-17T00:07:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:22 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 138/1500 objects degraded (9.200%), 5 pgs degraded) 2024-09-17T00:07:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:22 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:07:23.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 201866 8.86 MB/sec execute 427 sec latency 39.838 ms 2024-09-17T00:07:24.311 INFO:teuthology.orchestra.run.smithi007.stdout: 5 202182 8.87 MB/sec execute 428 sec latency 45.015 ms 2024-09-17T00:07:25.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 202418 8.87 MB/sec execute 429 sec latency 72.798 ms 2024-09-17T00:07:25.955 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:25.955 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:25.955 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:25 smithi007 ceph-mon[65487]: pgmap v181: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.4 MiB/s wr, 331 op/s; 344 KiB/s, 33 objects/s recovering 2024-09-17T00:07:26.063 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:26.063 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:26.063 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:25 smithi194 ceph-mon[51517]: pgmap v181: 97 pgs: 97 active+clean; 334 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.4 MiB/s wr, 331 op/s; 344 KiB/s, 33 objects/s recovering 2024-09-17T00:07:26.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 202657 8.86 MB/sec execute 430 sec latency 60.518 ms 2024-09-17T00:07:27.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 202833 8.84 MB/sec execute 431 sec latency 110.772 ms 2024-09-17T00:07:28.004 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:27 smithi007 ceph-mon[65487]: pgmap v182: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 5.0 MiB/s wr, 522 op/s; 261 KiB/s, 25 objects/s recovering 2024-09-17T00:07:28.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:27 smithi194 ceph-mon[51517]: pgmap v182: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.9 KiB/s rd, 5.0 MiB/s wr, 522 op/s; 261 KiB/s, 25 objects/s recovering 2024-09-17T00:07:28.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 203068 8.83 MB/sec execute 432 sec latency 63.038 ms 2024-09-17T00:07:29.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:28 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:29.047 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:28 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:29.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 203349 8.83 MB/sec execute 433 sec latency 70.115 ms 2024-09-17T00:07:29.998 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:29 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:29.998 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:29 smithi007 ceph-mon[65487]: Upgrade: osd.4 is safe to restart 2024-09-17T00:07:29.998 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:29 smithi007 ceph-mon[65487]: pgmap v183: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 475 op/s; 230 KiB/s, 22 objects/s recovering 2024-09-17T00:07:29.998 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:30.055 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:29 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-17T00:07:30.056 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:29 smithi194 ceph-mon[51517]: Upgrade: osd.4 is safe to restart 2024-09-17T00:07:30.056 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:29 smithi194 ceph-mon[51517]: pgmap v183: 97 pgs: 97 active+clean; 347 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.9 MiB/s wr, 475 op/s; 230 KiB/s, 22 objects/s recovering 2024-09-17T00:07:30.056 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:30.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 203715 8.82 MB/sec execute 434 sec latency 62.849 ms 2024-09-17T00:07:31.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:31 smithi007 ceph-mon[65487]: Upgrade: Updating osd.4 2024-09-17T00:07:31.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:31.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T00:07:31.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:07:31.300 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:31 smithi007 ceph-mon[65487]: Deploying daemon osd.4 on smithi194 2024-09-17T00:07:31.301 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:31 smithi007 ceph-mon[65487]: pgmap v184: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 477 op/s; 146 KiB/s, 18 objects/s recovering 2024-09-17T00:07:31.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 204056 8.82 MB/sec execute 435 sec latency 48.120 ms 2024-09-17T00:07:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:31 smithi194 ceph-mon[51517]: Upgrade: Updating osd.4 2024-09-17T00:07:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-17T00:07:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:07:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:31 smithi194 ceph-mon[51517]: Deploying daemon osd.4 on smithi194 2024-09-17T00:07:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:31 smithi194 ceph-mon[51517]: pgmap v184: 97 pgs: 97 active+clean; 341 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.6 MiB/s wr, 477 op/s; 146 KiB/s, 18 objects/s recovering 2024-09-17T00:07:32.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 204316 8.81 MB/sec execute 436 sec latency 60.561 ms 2024-09-17T00:07:33.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 204667 8.81 MB/sec execute 437 sec latency 73.218 ms 2024-09-17T00:07:33.898 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:33 smithi007 ceph-mon[65487]: pgmap v185: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 122 KiB/s, 11 objects/s recovering 2024-09-17T00:07:34.025 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:33 smithi194 ceph-mon[51517]: pgmap v185: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.2 MiB/s wr, 402 op/s; 122 KiB/s, 11 objects/s recovering 2024-09-17T00:07:34.026 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:33 smithi194 systemd[1]: Stopping Ceph osd.4 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:07:34.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 205003 8.82 MB/sec execute 438 sec latency 59.949 ms 2024-09-17T00:07:34.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:34 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[29939]: 2024-09-17T00:07:34.026+0000 7f6491d92700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:07:34.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:34 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[29939]: 2024-09-17T00:07:34.026+0000 7f6491d92700 -1 osd.4 115 *** Got signal Terminated *** 2024-09-17T00:07:34.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:34 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[29939]: 2024-09-17T00:07:34.026+0000 7f6491d92700 -1 osd.4 115 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:07:34.919 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:34 smithi007 ceph-mon[65487]: osd.4 marked itself down and dead 2024-09-17T00:07:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:34 smithi194 ceph-mon[51517]: osd.4 marked itself down and dead 2024-09-17T00:07:35.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 205168 8.81 MB/sec execute 439 sec latency 518.129 ms 2024-09-17T00:07:35.575 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:35 smithi194 podman[55933]: 2024-09-17 00:07:35.317373808 +0000 UTC m=+1.418422326 container died cbcef5eb9289203b960fad9a873d4c6cb74741725b33ae7dd079df1810666548 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, CEPH_POINT_RELEASE=-17.2.0, version=8, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, name=centos-stream, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, build-date=2022-05-03T08:36:31.336870) 2024-09-17T00:07:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:35 smithi194 ceph-mon[51517]: pgmap v186: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 375 op/s 2024-09-17T00:07:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:35 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:07:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:35 smithi194 ceph-mon[51517]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T00:07:35.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:35 smithi194 podman[55933]: 2024-09-17 00:07:35.785266788 +0000 UTC m=+1.886315305 container cleanup cbcef5eb9289203b960fad9a873d4c6cb74741725b33ae7dd079df1810666548 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, name=centos-stream, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, ceph=True, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True) 2024-09-17T00:07:35.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:35 smithi194 bash[55933]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4 2024-09-17T00:07:36.043 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:35 smithi007 ceph-mon[65487]: pgmap v186: 97 pgs: 97 active+clean; 338 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 375 op/s 2024-09-17T00:07:36.043 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:35 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:07:36.043 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:35 smithi007 ceph-mon[65487]: osdmap e116: 8 total, 7 up, 8 in 2024-09-17T00:07:36.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 205326 8.81 MB/sec execute 440 sec latency 1036.260 ms 2024-09-17T00:07:36.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[55949]: 2024-09-17 00:07:36.053999339 +0000 UTC m=+0.739321019 container remove cbcef5eb9289203b960fad9a873d4c6cb74741725b33ae7dd079df1810666548 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , ceph=True, architecture=x86_64, name=centos-stream, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.buildah.version=1.19.8, GIT_CLEAN=True, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0) 2024-09-17T00:07:36.672 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[56037]: 2024-09-17 00:07:36.317421717 +0000 UTC m=+0.018425111 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:07:36.672 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[56037]: 2024-09-17 00:07:36.429903069 +0000 UTC m=+0.130906448 container create 05577b7701356d844e100f4cf74dd35debdb0eac3c22fdedba7191efbc6443bd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T00:07:36.810 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:36 smithi007 ceph-mon[65487]: osdmap e117: 8 total, 7 up, 8 in 2024-09-17T00:07:37.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:36 smithi194 ceph-mon[51517]: osdmap e117: 8 total, 7 up, 8 in 2024-09-17T00:07:37.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[56037]: 2024-09-17 00:07:36.675023549 +0000 UTC m=+0.376026936 container init 05577b7701356d844e100f4cf74dd35debdb0eac3c22fdedba7191efbc6443bd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:07:37.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[56037]: 2024-09-17 00:07:36.679691496 +0000 UTC m=+0.380694877 container start 05577b7701356d844e100f4cf74dd35debdb0eac3c22fdedba7191efbc6443bd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T00:07:37.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[56037]: 2024-09-17 00:07:36.925078349 +0000 UTC m=+0.626081736 container attach 05577b7701356d844e100f4cf74dd35debdb0eac3c22fdedba7191efbc6443bd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-09-17T00:07:37.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:36 smithi194 podman[56037]: 2024-09-17 00:07:36.925522159 +0000 UTC m=+0.626525536 container died 05577b7701356d844e100f4cf74dd35debdb0eac3c22fdedba7191efbc6443bd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:07:37.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 205643 8.82 MB/sec execute 441 sec latency 47.275 ms 2024-09-17T00:07:37.834 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:37 smithi007 ceph-mon[65487]: pgmap v189: 97 pgs: 30 peering, 4 stale+active+clean, 63 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 483 op/s 2024-09-17T00:07:37.834 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:37 smithi007 ceph-mon[65487]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-17T00:07:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:37 smithi194 ceph-mon[51517]: pgmap v189: 97 pgs: 30 peering, 4 stale+active+clean, 63 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 483 op/s 2024-09-17T00:07:37.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:37 smithi194 ceph-mon[51517]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-17T00:07:37.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:37 smithi194 podman[56060]: 2024-09-17 00:07:37.563650331 +0000 UTC m=+0.648810640 container remove 05577b7701356d844e100f4cf74dd35debdb0eac3c22fdedba7191efbc6443bd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T00:07:37.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:37 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-17T00:07:37.910 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:37 smithi194 systemd[1]: Stopped Ceph osd.4 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:07:37.910 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:37 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.4.service: Consumed 3min 19.307s CPU time. 2024-09-17T00:07:37.910 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:37 smithi194 systemd[1]: Starting Ceph osd.4 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:07:38.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 205885 8.82 MB/sec execute 442 sec latency 71.853 ms 2024-09-17T00:07:38.331 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 podman[56152]: 2024-09-17 00:07:37.943540056 +0000 UTC m=+0.019468383 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:07:38.332 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 podman[56152]: 2024-09-17 00:07:38.073606448 +0000 UTC m=+0.149534758 container create f35ad3aa1a41bccc13b5c2637b77b6b6395dde6090de37b4fc33e50944d3b8e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T00:07:38.332 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 podman[56152]: 2024-09-17 00:07:38.334470496 +0000 UTC m=+0.410398817 container init f35ad3aa1a41bccc13b5c2637b77b6b6395dde6090de37b4fc33e50944d3b8e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:07:38.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 podman[56152]: 2024-09-17 00:07:38.339491933 +0000 UTC m=+0.415420253 container start f35ad3aa1a41bccc13b5c2637b77b6b6395dde6090de37b4fc33e50944d3b8e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-17T00:07:38.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 podman[56152]: 2024-09-17 00:07:38.443194427 +0000 UTC m=+0.519122776 container attach f35ad3aa1a41bccc13b5c2637b77b6b6395dde6090de37b4fc33e50944d3b8e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:07:38.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:07:38.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 bash[56152]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:07:38.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:07:38.660 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:38 smithi194 bash[56152]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:07:39.103 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:38 smithi007 ceph-mon[65487]: pgmap v190: 97 pgs: 3 active+undersized, 30 peering, 8 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 383 op/s; 80/1602 objects degraded (4.994%) 2024-09-17T00:07:39.103 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:38 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 80/1602 objects degraded (4.994%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:39.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:38 smithi194 ceph-mon[51517]: pgmap v190: 97 pgs: 3 active+undersized, 30 peering, 8 active+undersized+degraded, 56 active+clean; 339 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 3.8 MiB/s wr, 383 op/s; 80/1602 objects degraded (4.994%) 2024-09-17T00:07:39.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:38 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 80/1602 objects degraded (4.994%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:39.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 206096 8.81 MB/sec execute 443 sec latency 83.460 ms 2024-09-17T00:07:39.545 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:07:40.158 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:39 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T00:07:40.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:39 smithi194 bash[56152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T00:07:40.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:39 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T00:07:40.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:39 smithi194 bash[56152]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-17T00:07:40.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 206307 8.80 MB/sec execute 444 sec latency 61.846 ms 2024-09-17T00:07:40.405 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (6m) 33s ago 15m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (3m) 33s ago 15m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (3m) 3m ago 15m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (6m) 33s ago 15m 53.4M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (9m) 33s ago 9m 37.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (9m) 3m ago 9m 14.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (6m) 33s ago 17m 503M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (5m) 3m ago 15m 439M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (4m) 33s ago 17m 50.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (4m) 3m ago 15m 32.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:07:40.406 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (9m) 33s ago 9m 270M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (9m) 3m ago 9m 88.0M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (6m) 33s ago 15m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 3m ago 15m 16.8M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (3m) 33s ago 14m 321M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (2m) 33s ago 13m 258M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (88s) 33s ago 13m 215M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (42s) 33s ago 13m 11.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (12m) 3m ago 12m 479M 2704M 17.2.0 e1d6a67b021e cbcef5eb9289 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (12m) 3m ago 12m 556M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (11m) 3m ago 11m 480M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (11m) 3m ago 11m 364M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:07:40.407 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (5m) 33s ago 14m 66.3M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:07:40.570 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T00:07:40.570 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 bash[56152]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 bash[56152]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 bash[56152]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 bash[56152]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate[56166]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 bash[56152]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-17T00:07:40.571 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 podman[56152]: 2024-09-17 00:07:40.203642471 +0000 UTC m=+2.279570782 container died f35ad3aa1a41bccc13b5c2637b77b6b6395dde6090de37b4fc33e50944d3b8e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-17T00:07:40.888 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-mon[51517]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:40.888 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:40.888 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:40.903 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:07:40.903 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 4 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:07:40.904 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:07:40.905 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-17T00:07:40.905 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T00:07:40.905 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:07:40.905 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:07:40.913 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:40 smithi007 ceph-mon[65487]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:40.913 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:40.913 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:41.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:40 smithi194 podman[56300]: 2024-09-17 00:07:40.890996408 +0000 UTC m=+0.681411542 container remove f35ad3aa1a41bccc13b5c2637b77b6b6395dde6090de37b4fc33e50944d3b8e3 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-activate, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-17T00:07:41.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 podman[56355]: 2024-09-17 00:07:41.015075495 +0000 UTC m=+0.019639510 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:07:41.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 podman[56355]: 2024-09-17 00:07:41.117123236 +0000 UTC m=+0.121687243 container create 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-17T00:07:41.310 INFO:teuthology.orchestra.run.smithi007.stdout: 5 206514 8.78 MB/sec execute 445 sec latency 54.184 ms 2024-09-17T00:07:41.381 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:07:41.381 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "10/23 daemons upgraded", 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:07:41.382 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:07:41.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 podman[56355]: 2024-09-17 00:07:41.370373178 +0000 UTC m=+0.374937226 container init 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:07:41.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 podman[56355]: 2024-09-17 00:07:41.373727804 +0000 UTC m=+0.378291810 container start 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-17T00:07:41.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-osd[56374]: -- 172.21.15.194:0/2814944555 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55efd04001e0 con 0x55efd03da000 2024-09-17T00:07:41.659 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 bash[56355]: 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 2024-09-17T00:07:41.660 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:41 smithi194 systemd[1]: Started Ceph osd.4 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: pgmap v191: 97 pgs: 9 active+undersized, 13 peering, 19 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 3.8 MiB/s wr, 408 op/s; 161/1656 objects degraded (9.722%) 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/784547031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:41.922 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:41 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 161/1656 objects degraded (9.722%), 19 pgs degraded 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: osd.4 (root=default,host=smithi194) is down 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 161/1656 objects degraded (9.722%), 19 pgs degraded 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-17T00:07:42.086 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-17T00:07:42.087 INFO:teuthology.orchestra.run.smithi007.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: from='client.44259 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: pgmap v191: 97 pgs: 9 active+undersized, 13 peering, 19 active+undersized+degraded, 56 active+clean; 342 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.4 KiB/s rd, 3.8 MiB/s wr, 408 op/s; 161/1656 objects degraded (9.722%) 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/784547031' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:42.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:41 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:42.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 206765 8.77 MB/sec execute 446 sec latency 75.757 ms 2024-09-17T00:07:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:42 smithi194 ceph-mon[51517]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:42 smithi194 ceph-mon[51517]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-17T00:07:42.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:42 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/4182360658' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:07:42.965 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:42 smithi007 ceph-mon[65487]: from='client.34350 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:07:42.965 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:42 smithi007 ceph-mon[65487]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-17T00:07:42.965 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:42 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/4182360658' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:07:43.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 207148 8.78 MB/sec execute 447 sec latency 61.753 ms 2024-09-17T00:07:43.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:43 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:07:43.269+0000 7fdee0c72640 -1 Falling back to public interface 2024-09-17T00:07:43.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:43 smithi007 ceph-mon[65487]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.2 MiB/s wr, 458 op/s; 262/1674 objects degraded (15.651%) 2024-09-17T00:07:43.996 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:43 smithi194 ceph-mon[51517]: pgmap v192: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 5.2 MiB/s wr, 458 op/s; 262/1674 objects degraded (15.651%) 2024-09-17T00:07:44.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 207559 8.77 MB/sec execute 448 sec latency 39.693 ms 2024-09-17T00:07:44.742 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:44 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 262/1674 objects degraded (15.651%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:45.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:44 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 262/1674 objects degraded (15.651%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:45.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 207889 8.77 MB/sec execute 449 sec latency 65.916 ms 2024-09-17T00:07:45.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:45 smithi007 ceph-mon[65487]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 368 op/s; 262/1674 objects degraded (15.651%) 2024-09-17T00:07:45.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:45.804 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:46.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:45 smithi194 ceph-mon[51517]: pgmap v193: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 345 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 368 op/s; 262/1674 objects degraded (15.651%) 2024-09-17T00:07:46.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:46.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:46.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 208231 8.77 MB/sec execute 450 sec latency 40.183 ms 2024-09-17T00:07:47.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 208542 8.78 MB/sec execute 451 sec latency 57.782 ms 2024-09-17T00:07:47.329 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:47.329 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:47.329 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:47 smithi007 ceph-mon[65487]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.8 MiB/s wr, 390 op/s; 244/1593 objects degraded (15.317%) 2024-09-17T00:07:47.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:47.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:47.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:47 smithi194 ceph-mon[51517]: pgmap v194: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 337 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.8 MiB/s wr, 390 op/s; 244/1593 objects degraded (15.317%) 2024-09-17T00:07:48.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 208867 8.79 MB/sec execute 452 sec latency 49.932 ms 2024-09-17T00:07:49.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 209187 8.80 MB/sec execute 453 sec latency 72.645 ms 2024-09-17T00:07:49.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:49 smithi007 ceph-mon[65487]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 376 op/s; 242/1584 objects degraded (15.278%) 2024-09-17T00:07:49.893 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:49 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 242/1584 objects degraded (15.278%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:49.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:49 smithi194 ceph-mon[51517]: pgmap v195: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 336 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.7 MiB/s wr, 376 op/s; 242/1584 objects degraded (15.278%) 2024-09-17T00:07:49.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:49 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 242/1584 objects degraded (15.278%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:50.159 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:07:49.909+0000 7fdee0c72640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-17T00:07:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 209396 8.79 MB/sec execute 454 sec latency 54.253 ms 2024-09-17T00:07:51.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 209565 8.79 MB/sec execute 455 sec latency 54.352 ms 2024-09-17T00:07:51.408 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:50 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:07:50.929+0000 7fdee0c72640 -1 osd.4 115 log_to_monitors true 2024-09-17T00:07:51.450 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.451 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.451 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:07:51.452 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:07:51.452 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.452 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.452 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:51.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:51.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:51.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:51.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:07:51.453 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:07:51.454 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T00:07:51.454 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 377 op/s; 240/1578 objects degraded (15.209%) 2024-09-17T00:07:51.454 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='osd.4 [v2:172.21.15.194:6800/279754613,v1:172.21.15.194:6801/279754613]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:07:51.454 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:51 smithi007 ceph-mon[65487]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:07:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:07:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:07:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:51.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:07:51.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:51.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:51.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:07:51.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:07:51.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:07:51.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2024-09-17T00:07:51.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: pgmap v196: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 335 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.7 MiB/s wr, 377 op/s; 240/1578 objects degraded (15.209%) 2024-09-17T00:07:51.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='osd.4 [v2:172.21.15.194:6800/279754613,v1:172.21.15.194:6801/279754613]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:07:51.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-mon[51517]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-17T00:07:52.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 209730 8.77 MB/sec execute 456 sec latency 153.185 ms 2024-09-17T00:07:52.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:07:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:07:51.940+0000 7fded4a15640 -1 osd.4 115 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:07:52.511 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:52 smithi007 ceph-mon[65487]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T00:07:52.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:52 smithi007 ceph-mon[65487]: from='osd.4 [v2:172.21.15.194:6800/279754613,v1:172.21.15.194:6801/279754613]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:07:52.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:52 smithi007 ceph-mon[65487]: osdmap e118: 8 total, 7 up, 8 in 2024-09-17T00:07:52.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:52 smithi007 ceph-mon[65487]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:07:52.512 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:52 smithi007 ceph-mon[65487]: from='osd.4 ' entity='osd.4' 2024-09-17T00:07:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:52 smithi194 ceph-mon[51517]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-17T00:07:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:52 smithi194 ceph-mon[51517]: from='osd.4 [v2:172.21.15.194:6800/279754613,v1:172.21.15.194:6801/279754613]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:07:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:52 smithi194 ceph-mon[51517]: osdmap e118: 8 total, 7 up, 8 in 2024-09-17T00:07:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:52 smithi194 ceph-mon[51517]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:07:52.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:52 smithi194 ceph-mon[51517]: from='osd.4 ' entity='osd.4' 2024-09-17T00:07:53.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 209934 8.76 MB/sec execute 457 sec latency 350.804 ms 2024-09-17T00:07:53.527 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:53 smithi007 ceph-mon[65487]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 451 op/s; 255/1686 objects degraded (15.125%) 2024-09-17T00:07:53.527 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:53 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:07:53.527 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:53 smithi007 ceph-mon[65487]: osd.4 [v2:172.21.15.194:6800/279754613,v1:172.21.15.194:6801/279754613] boot 2024-09-17T00:07:53.527 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:53 smithi007 ceph-mon[65487]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T00:07:53.527 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:53 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:07:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:53 smithi194 ceph-mon[51517]: pgmap v198: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.5 MiB/s wr, 451 op/s; 255/1686 objects degraded (15.125%) 2024-09-17T00:07:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:53 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:07:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:53 smithi194 ceph-mon[51517]: osd.4 [v2:172.21.15.194:6800/279754613,v1:172.21.15.194:6801/279754613] boot 2024-09-17T00:07:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:53 smithi194 ceph-mon[51517]: osdmap e119: 8 total, 8 up, 8 in 2024-09-17T00:07:53.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:53 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-17T00:07:54.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 210048 8.74 MB/sec execute 458 sec latency 1080.539 ms 2024-09-17T00:07:55.060 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:54 smithi007 ceph-mon[65487]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T00:07:55.060 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:54 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 255/1686 objects degraded (15.125%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:55.060 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:55.060 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:55.060 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:54 smithi007 ceph-mon[65487]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 333 op/s; 255/1686 objects degraded (15.125%) 2024-09-17T00:07:55.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 210223 8.73 MB/sec execute 459 sec latency 59.867 ms 2024-09-17T00:07:55.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:54 smithi194 ceph-mon[51517]: osdmap e120: 8 total, 8 up, 8 in 2024-09-17T00:07:55.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:54 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 255/1686 objects degraded (15.125%), 28 pgs degraded (PG_DEGRADED) 2024-09-17T00:07:55.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:07:55.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:07:55.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:54 smithi194 ceph-mon[51517]: pgmap v201: 97 pgs: 13 active+undersized, 28 active+undersized+degraded, 56 active+clean; 343 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 4.4 MiB/s wr, 333 op/s; 255/1686 objects degraded (15.125%) 2024-09-17T00:07:56.077 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:55 smithi007 ceph-mon[65487]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T00:07:56.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 210418 8.72 MB/sec execute 460 sec latency 72.879 ms 2024-09-17T00:07:56.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:55 smithi194 ceph-mon[51517]: osdmap e121: 8 total, 8 up, 8 in 2024-09-17T00:07:57.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:56 smithi007 ceph-mon[65487]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T00:07:57.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:56 smithi007 ceph-mon[65487]: pgmap v204: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 2 active+undersized+remapped, 6 active+recovery_wait+degraded, 2 active+undersized, 5 active+undersized+degraded, 78 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 341 op/s; 182/1515 objects degraded (12.013%); 2.2 MiB/s, 59 keys/s, 28 objects/s recovering 2024-09-17T00:07:57.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 210638 8.71 MB/sec execute 461 sec latency 78.450 ms 2024-09-17T00:07:57.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:56 smithi194 ceph-mon[51517]: osdmap e122: 8 total, 8 up, 8 in 2024-09-17T00:07:57.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:56 smithi194 ceph-mon[51517]: pgmap v204: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 2 active+undersized+remapped, 6 active+recovery_wait+degraded, 2 active+undersized, 5 active+undersized+degraded, 78 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.6 MiB/s wr, 341 op/s; 182/1515 objects degraded (12.013%); 2.2 MiB/s, 59 keys/s, 28 objects/s recovering 2024-09-17T00:07:58.123 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:57 smithi007 ceph-mon[65487]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T00:07:58.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 210994 8.72 MB/sec execute 462 sec latency 347.573 ms 2024-09-17T00:07:58.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:57 smithi194 ceph-mon[51517]: osdmap e123: 8 total, 8 up, 8 in 2024-09-17T00:07:59.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:58 smithi007 ceph-mon[65487]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T00:07:59.135 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:58 smithi007 ceph-mon[65487]: pgmap v207: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 3 active+undersized+remapped, 6 active+recovery_wait+degraded, 84 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 MiB/s wr, 378 op/s; 128/1449 objects degraded (8.834%); 2.4 MiB/s, 59 keys/s, 44 objects/s recovering 2024-09-17T00:07:59.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 211386 8.71 MB/sec execute 463 sec latency 453.893 ms 2024-09-17T00:07:59.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:58 smithi194 ceph-mon[51517]: osdmap e124: 8 total, 8 up, 8 in 2024-09-17T00:07:59.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:58 smithi194 ceph-mon[51517]: pgmap v207: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded, 3 active+undersized+remapped, 6 active+recovery_wait+degraded, 84 active+clean; 335 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 3.2 MiB/s wr, 378 op/s; 128/1449 objects degraded (8.834%); 2.4 MiB/s, 59 keys/s, 44 objects/s recovering 2024-09-17T00:08:00.188 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:59 smithi007 ceph-mon[65487]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T00:08:00.188 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:07:59 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 128/1449 objects degraded (8.834%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:00.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 211659 8.70 MB/sec execute 464 sec latency 722.977 ms 2024-09-17T00:08:00.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:59 smithi194 ceph-mon[51517]: osdmap e125: 8 total, 8 up, 8 in 2024-09-17T00:08:00.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:07:59 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 128/1449 objects degraded (8.834%), 10 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:01.252 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:01 smithi007 ceph-mon[65487]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T00:08:01.252 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:01 smithi007 ceph-mon[65487]: pgmap v210: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+degraded, 89 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.4 MiB/s wr, 407 op/s; 104/1467 objects degraded (7.089%); 181 KiB/s, 21 objects/s recovering 2024-09-17T00:08:01.310 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:01 smithi194 ceph-mon[51517]: osdmap e126: 8 total, 8 up, 8 in 2024-09-17T00:08:01.310 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:01 smithi194 ceph-mon[51517]: pgmap v210: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+degraded, 89 active+clean; 338 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 2.4 MiB/s wr, 407 op/s; 104/1467 objects degraded (7.089%); 181 KiB/s, 21 objects/s recovering 2024-09-17T00:08:01.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 211900 8.70 MB/sec execute 465 sec latency 48.291 ms 2024-09-17T00:08:02.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 212328 8.72 MB/sec execute 466 sec latency 35.371 ms 2024-09-17T00:08:03.062 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:02 smithi007 ceph-mon[65487]: pgmap v211: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.0 MiB/s wr, 361 op/s; 117 KiB/s, 22 objects/s recovering 2024-09-17T00:08:03.062 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:02 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 104/1467 objects degraded (7.089%), 7 pgs degraded) 2024-09-17T00:08:03.062 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:02 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:08:03.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:02 smithi194 ceph-mon[51517]: pgmap v211: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.8 KiB/s rd, 3.0 MiB/s wr, 361 op/s; 117 KiB/s, 22 objects/s recovering 2024-09-17T00:08:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:02 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 104/1467 objects degraded (7.089%), 7 pgs degraded) 2024-09-17T00:08:03.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:02 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:08:03.312 INFO:teuthology.orchestra.run.smithi007.stdout: 5 212741 8.74 MB/sec execute 467 sec latency 35.771 ms 2024-09-17T00:08:04.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 213035 8.75 MB/sec execute 468 sec latency 78.367 ms 2024-09-17T00:08:05.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 213271 8.73 MB/sec execute 469 sec latency 56.503 ms 2024-09-17T00:08:05.833 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:05 smithi007 ceph-mon[65487]: pgmap v212: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 308 op/s; 99 KiB/s, 18 objects/s recovering 2024-09-17T00:08:05.833 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:05 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:08:06.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:05 smithi194 ceph-mon[51517]: pgmap v212: 97 pgs: 97 active+clean; 331 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.6 MiB/s wr, 308 op/s; 99 KiB/s, 18 objects/s recovering 2024-09-17T00:08:06.114 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:05 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:08:06.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 213600 8.72 MB/sec execute 470 sec latency 48.963 ms 2024-09-17T00:08:06.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:06 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:08:06.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:06 smithi007 ceph-mon[65487]: Upgrade: osd.5 is safe to restart 2024-09-17T00:08:07.137 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:06 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-17T00:08:07.137 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:06 smithi194 ceph-mon[51517]: Upgrade: osd.5 is safe to restart 2024-09-17T00:08:07.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 213894 8.72 MB/sec execute 471 sec latency 37.299 ms 2024-09-17T00:08:07.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:07 smithi007 ceph-mon[65487]: pgmap v213: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.1 MiB/s wr, 527 op/s; 84 KiB/s, 16 objects/s recovering 2024-09-17T00:08:07.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:07 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:07.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:07 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T00:08:07.874 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:07 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:08:08.153 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:07 smithi194 ceph-mon[51517]: pgmap v213: 97 pgs: 97 active+clean; 345 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 5.1 MiB/s wr, 527 op/s; 84 KiB/s, 16 objects/s recovering 2024-09-17T00:08:08.154 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:07 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:08.154 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:07 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-17T00:08:08.154 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:07 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:08:08.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 214259 8.72 MB/sec execute 472 sec latency 40.764 ms 2024-09-17T00:08:08.902 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:08 smithi007 ceph-mon[65487]: Upgrade: Updating osd.5 2024-09-17T00:08:08.903 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:08 smithi007 ceph-mon[65487]: Deploying daemon osd.5 on smithi194 2024-09-17T00:08:09.156 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:08 smithi194 ceph-mon[51517]: Upgrade: Updating osd.5 2024-09-17T00:08:09.157 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:08 smithi194 ceph-mon[51517]: Deploying daemon osd.5 on smithi194 2024-09-17T00:08:09.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 214569 8.71 MB/sec execute 473 sec latency 61.900 ms 2024-09-17T00:08:09.924 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:09 smithi007 ceph-mon[65487]: pgmap v214: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 451 op/s; 69 KiB/s, 13 objects/s recovering 2024-09-17T00:08:09.924 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:09.924 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:09.924 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:09 smithi194 ceph-mon[51517]: pgmap v214: 97 pgs: 97 active+clean; 344 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.8 MiB/s wr, 451 op/s; 69 KiB/s, 13 objects/s recovering 2024-09-17T00:08:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:10.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 214941 8.70 MB/sec execute 474 sec latency 55.988 ms 2024-09-17T00:08:10.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:10 smithi194 ceph-mon[51517]: pgmap v215: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 493 op/s; 63 KiB/s, 11 objects/s recovering 2024-09-17T00:08:10.977 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:10 smithi007 ceph-mon[65487]: pgmap v215: 97 pgs: 97 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.7 MiB/s wr, 493 op/s; 63 KiB/s, 11 objects/s recovering 2024-09-17T00:08:11.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 215284 8.70 MB/sec execute 475 sec latency 46.804 ms 2024-09-17T00:08:11.372 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:11 smithi194 systemd[1]: Stopping Ceph osd.5 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:08:11.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[32896]: 2024-09-17T00:08:11.372+0000 7f18f629b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:08:11.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[32896]: 2024-09-17T00:08:11.372+0000 7f18f629b700 -1 osd.5 126 *** Got signal Terminated *** 2024-09-17T00:08:11.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[32896]: 2024-09-17T00:08:11.372+0000 7f18f629b700 -1 osd.5 126 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:08:12.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:11 smithi007 ceph-mon[65487]: osd.5 marked itself down and dead 2024-09-17T00:08:12.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:11 smithi194 ceph-mon[51517]: osd.5 marked itself down and dead 2024-09-17T00:08:12.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 215379 8.69 MB/sec execute 476 sec latency 456.234 ms 2024-09-17T00:08:12.463 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:08:12.795 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:12 smithi194 podman[58185]: 2024-09-17 00:08:12.515600247 +0000 UTC m=+1.268929080 container died 3fd3c3067162d34e4e17d7132b506a34df715f6bd57a5846ecf51edc7f701413 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, release=754, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD) 2024-09-17T00:08:13.128 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:12 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:08:13.128 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:12 smithi007 ceph-mon[65487]: osdmap e127: 8 total, 7 up, 8 in 2024-09-17T00:08:13.128 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:12 smithi007 ceph-mon[65487]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:13.128 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:12 smithi007 ceph-mon[65487]: pgmap v217: 97 pgs: 12 stale+active+clean, 85 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-09-17T00:08:13.128 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:12 smithi007 ceph-mon[65487]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:13.154 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:12 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:08:13.155 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:12 smithi194 ceph-mon[51517]: osdmap e127: 8 total, 7 up, 8 in 2024-09-17T00:08:13.155 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:12 smithi194 ceph-mon[51517]: from='client.34360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:13.155 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:12 smithi194 ceph-mon[51517]: pgmap v217: 97 pgs: 12 stale+active+clean, 85 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.7 MiB/s wr, 482 op/s 2024-09-17T00:08:13.155 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:12 smithi194 ceph-mon[51517]: from='client.34364 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:13.155 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:12 smithi194 podman[58185]: 2024-09-17 00:08:12.897352553 +0000 UTC m=+1.650681377 container cleanup 3fd3c3067162d34e4e17d7132b506a34df715f6bd57a5846ecf51edc7f701413 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , architecture=x86_64, ceph=True, name=centos-stream) 2024-09-17T00:08:13.155 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:12 smithi194 bash[58185]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5 2024-09-17T00:08:13.246 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (7m) 66s ago 16m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (4m) 66s ago 16m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (4m) 27s ago 15m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (7m) 66s ago 16m 53.4M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (10m) 66s ago 10m 37.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (10m) 27s ago 10m 15.2M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (6m) 66s ago 17m 503M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (5m) 27s ago 15m 442M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (5m) 66s ago 17m 50.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:08:13.247 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (5m) 27s ago 15m 39.7M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (10m) 66s ago 10m 270M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (10m) 27s ago 10m 88.5M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (7m) 66s ago 16m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (6m) 27s ago 15m 16.6M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (3m) 66s ago 14m 321M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (2m) 66s ago 14m 258M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (2m) 66s ago 14m 215M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (75s) 66s ago 13m 11.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (32s) 27s ago 13m 14.4M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (12m) 27s ago 12m 492M 2704M 17.2.0 e1d6a67b021e 3fd3c3067162 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (12m) 27s ago 12m 397M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (12m) 27s ago 12m 598M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:08:13.248 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (5m) 66s ago 15m 66.3M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:08:13.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 215460 8.68 MB/sec execute 477 sec latency 1053.238 ms 2024-09-17T00:08:13.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:13 smithi194 podman[58199]: 2024-09-17 00:08:13.157443168 +0000 UTC m=+0.639074072 container remove 3fd3c3067162d34e4e17d7132b506a34df715f6bd57a5846ecf51edc7f701413 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, distribution-scope=public, GIT_BRANCH=HEAD, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-09-17T00:08:13.861 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:08:13.861 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:08:13.861 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:08:13.861 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:13.861 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 5 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 9 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:08:13.862 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:08:13.891 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:13 smithi194 podman[58287]: 2024-09-17 00:08:13.406435589 +0000 UTC m=+0.019124612 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:08:13.891 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:13 smithi194 podman[58287]: 2024-09-17 00:08:13.533571763 +0000 UTC m=+0.146260778 container create eaf15b1b950839e3a3b08421c4ea51657b2ab5b5262ea55dd209af4bfb21ea01 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-17T00:08:13.895 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:13 smithi007 ceph-mon[65487]: osdmap e128: 8 total, 7 up, 8 in 2024-09-17T00:08:13.896 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:13 smithi007 ceph-mon[65487]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:13.896 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:13 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3929865752' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:13 smithi194 ceph-mon[51517]: osdmap e128: 8 total, 7 up, 8 in 2024-09-17T00:08:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:13 smithi194 ceph-mon[51517]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:14.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:13 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3929865752' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:14.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 215823 8.70 MB/sec execute 478 sec latency 71.775 ms 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:08:14.380 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "11/23 daemons upgraded", 2024-09-17T00:08:14.381 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:08:14.381 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:08:14.381 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:08:14.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:14 smithi194 podman[58287]: 2024-09-17 00:08:14.170225214 +0000 UTC m=+0.782914235 container init eaf15b1b950839e3a3b08421c4ea51657b2ab5b5262ea55dd209af4bfb21ea01 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T00:08:14.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:14 smithi194 podman[58287]: 2024-09-17 00:08:14.173661544 +0000 UTC m=+0.786350554 container start eaf15b1b950839e3a3b08421c4ea51657b2ab5b5262ea55dd209af4bfb21ea01 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T00:08:14.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:14 smithi194 podman[58287]: 2024-09-17 00:08:14.245126155 +0000 UTC m=+0.857815195 container attach eaf15b1b950839e3a3b08421c4ea51657b2ab5b5262ea55dd209af4bfb21ea01 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True) 2024-09-17T00:08:14.660 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:14 smithi194 podman[58287]: 2024-09-17 00:08:14.38845734 +0000 UTC m=+1.001146362 container died eaf15b1b950839e3a3b08421c4ea51657b2ab5b5262ea55dd209af4bfb21ea01 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:08:14.908 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_WARN 1 osds down 2024-09-17T00:08:14.908 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-17T00:08:14.908 INFO:teuthology.orchestra.run.smithi007.stdout: osd.5 (root=default,host=smithi194) is down 2024-09-17T00:08:14.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:14 smithi007 ceph-mon[65487]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:14.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:14 smithi007 ceph-mon[65487]: pgmap v219: 97 pgs: 12 stale+active+clean, 85 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.9 MiB/s wr, 337 op/s 2024-09-17T00:08:15.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 216175 8.70 MB/sec execute 479 sec latency 41.815 ms 2024-09-17T00:08:15.314 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:14 smithi194 ceph-mon[51517]: from='client.34376 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:15.314 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:14 smithi194 ceph-mon[51517]: pgmap v219: 97 pgs: 12 stale+active+clean, 85 active+clean; 328 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.9 MiB/s wr, 337 op/s 2024-09-17T00:08:15.565 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 podman[58310]: 2024-09-17 00:08:15.316936469 +0000 UTC m=+0.922513215 container remove eaf15b1b950839e3a3b08421c4ea51657b2ab5b5262ea55dd209af4bfb21ea01 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-17T00:08:15.565 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-17T00:08:15.565 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 systemd[1]: Stopped Ceph osd.5 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:08:15.565 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 14.441s CPU time. 2024-09-17T00:08:15.816 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 systemd[1]: Starting Ceph osd.5 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:08:15.816 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 podman[58409]: 2024-09-17 00:08:15.695056932 +0000 UTC m=+0.017594746 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:08:15.816 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:15 smithi194 podman[58409]: 2024-09-17 00:08:15.81870216 +0000 UTC m=+0.141239970 container create c8e6cc7510fe5cf640b4d48831394aa2ae200406df53e61970c4f20980fa9c7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:08:16.003 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:15 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/81922587' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:08:16.068 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:15 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/81922587' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:08:16.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 216489 8.71 MB/sec execute 480 sec latency 70.556 ms 2024-09-17T00:08:16.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 podman[58409]: 2024-09-17 00:08:16.071876665 +0000 UTC m=+0.394414477 container init c8e6cc7510fe5cf640b4d48831394aa2ae200406df53e61970c4f20980fa9c7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T00:08:16.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 podman[58409]: 2024-09-17 00:08:16.076316997 +0000 UTC m=+0.398854803 container start c8e6cc7510fe5cf640b4d48831394aa2ae200406df53e61970c4f20980fa9c7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:08:16.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 podman[58409]: 2024-09-17 00:08:16.17184924 +0000 UTC m=+0.494387040 container attach c8e6cc7510fe5cf640b4d48831394aa2ae200406df53e61970c4f20980fa9c7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:08:16.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:16.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 bash[58409]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:16.410 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:16.410 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:16 smithi194 bash[58409]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:17.011 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:16 smithi007 ceph-mon[65487]: pgmap v220: 97 pgs: 6 active+undersized, 1 stale+active+clean, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 502 op/s; 153/1656 objects degraded (9.239%) 2024-09-17T00:08:17.307 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:16 smithi194 ceph-mon[51517]: pgmap v220: 97 pgs: 6 active+undersized, 1 stale+active+clean, 19 active+undersized+degraded, 71 active+clean; 336 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 502 op/s; 153/1656 objects degraded (9.239%) 2024-09-17T00:08:17.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 216725 8.71 MB/sec execute 481 sec latency 52.734 ms 2024-09-17T00:08:17.593 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T00:08:17.593 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T00:08:17.593 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T00:08:17.593 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-17T00:08:17.593 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T00:08:17.594 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-17T00:08:17.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T00:08:17.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-17T00:08:17.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T00:08:17.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-17T00:08:17.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T00:08:17.909 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-17T00:08:17.910 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate[58422]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T00:08:17.910 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 bash[58409]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-17T00:08:17.910 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:17 smithi194 podman[58409]: 2024-09-17 00:08:17.632178523 +0000 UTC m=+1.954716333 container died c8e6cc7510fe5cf640b4d48831394aa2ae200406df53e61970c4f20980fa9c7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97) 2024-09-17T00:08:18.033 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:17 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 153/1656 objects degraded (9.239%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:18.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 216903 8.70 MB/sec execute 482 sec latency 56.991 ms 2024-09-17T00:08:18.341 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:17 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 153/1656 objects degraded (9.239%), 19 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:18.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 podman[58549]: 2024-09-17 00:08:18.343979544 +0000 UTC m=+0.702218286 container remove c8e6cc7510fe5cf640b4d48831394aa2ae200406df53e61970c4f20980fa9c7f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-17T00:08:18.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 podman[58608]: 2024-09-17 00:08:18.453848713 +0000 UTC m=+0.015885119 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:08:18.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 podman[58608]: 2024-09-17 00:08:18.586703716 +0000 UTC m=+0.148740106 container create cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=) 2024-09-17T00:08:18.974 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 podman[58608]: 2024-09-17 00:08:18.889344426 +0000 UTC m=+0.451380827 container init cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:08:18.974 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 podman[58608]: 2024-09-17 00:08:18.892736465 +0000 UTC m=+0.454772869 container start cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-17T00:08:18.974 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 ceph-osd[58627]: -- 172.21.15.194:0/1727067005 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561eec0ea1e0 con 0x561eec0c8000 2024-09-17T00:08:19.038 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:18 smithi007 ceph-mon[65487]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 406 op/s; 190/1674 objects degraded (11.350%) 2024-09-17T00:08:19.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 217133 8.68 MB/sec execute 483 sec latency 55.241 ms 2024-09-17T00:08:19.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:18 smithi194 ceph-mon[51517]: pgmap v221: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 338 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.6 MiB/s wr, 406 op/s; 190/1674 objects degraded (11.350%) 2024-09-17T00:08:19.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 bash[58608]: cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 2024-09-17T00:08:19.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:18 smithi194 systemd[1]: Started Ceph osd.5 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:08:20.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:20.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:20.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:08:20.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 217344 8.67 MB/sec execute 484 sec latency 46.718 ms 2024-09-17T00:08:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:20.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:08:21.023 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:20 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:08:20.767+0000 7ffbe5ced640 -1 Falling back to public interface 2024-09-17T00:08:21.089 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:21 smithi007 ceph-mon[65487]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.1 MiB/s wr, 260 op/s; 189/1689 objects degraded (11.190%) 2024-09-17T00:08:21.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 217559 8.67 MB/sec execute 485 sec latency 68.776 ms 2024-09-17T00:08:21.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:21 smithi194 ceph-mon[51517]: pgmap v222: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.3 KiB/s rd, 3.1 MiB/s wr, 260 op/s; 189/1689 objects degraded (11.190%) 2024-09-17T00:08:22.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 217802 8.66 MB/sec execute 486 sec latency 67.599 ms 2024-09-17T00:08:22.923 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:22 smithi007 ceph-mon[65487]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.1 MiB/s wr, 371 op/s; 175/1587 objects degraded (11.027%) 2024-09-17T00:08:22.923 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:22 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 175/1587 objects degraded (11.027%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:23.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:22 smithi194 ceph-mon[51517]: pgmap v223: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.1 MiB/s wr, 371 op/s; 175/1587 objects degraded (11.027%) 2024-09-17T00:08:23.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:22 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 175/1587 objects degraded (11.027%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:23.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 218039 8.66 MB/sec execute 487 sec latency 119.892 ms 2024-09-17T00:08:23.949 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:23.949 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:23.949 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:23.949 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:23 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:24.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 218295 8.65 MB/sec execute 488 sec latency 80.306 ms 2024-09-17T00:08:24.348 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:24.349 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:24.349 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:24.349 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:23 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:25.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 218568 8.64 MB/sec execute 489 sec latency 54.137 ms 2024-09-17T00:08:25.757 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:25.757 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:25.757 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:25 smithi007 ceph-mon[65487]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 312 op/s; 175/1587 objects degraded (11.027%) 2024-09-17T00:08:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:25.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:25 smithi194 ceph-mon[51517]: pgmap v224: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 331 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 312 op/s; 175/1587 objects degraded (11.027%) 2024-09-17T00:08:26.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 218844 8.63 MB/sec execute 490 sec latency 54.025 ms 2024-09-17T00:08:27.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 219035 8.63 MB/sec execute 491 sec latency 83.895 ms 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: pgmap v225: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 322 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 435 op/s; 170/1512 objects degraded (11.243%) 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.855 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:27 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: pgmap v225: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 322 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.3 MiB/s wr, 435 op/s; 170/1512 objects degraded (11.243%) 2024-09-17T00:08:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:08:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:08:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:27.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:27 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:28.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 219355 8.64 MB/sec execute 492 sec latency 52.242 ms 2024-09-17T00:08:28.819 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:08:28.501+0000 7ffbe5ced640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-17T00:08:28.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:28 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:08:28.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:28 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:28.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:28 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:28.906 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:28 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:28.907 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:28 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:28 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:08:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:28 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:28 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:28 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:29.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:28 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:29.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 219686 8.65 MB/sec execute 493 sec latency 42.512 ms 2024-09-17T00:08:29.908 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:08:29.659+0000 7ffbe5ced640 -1 osd.5 126 log_to_monitors true 2024-09-17T00:08:29.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:29.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-mon[51517]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T00:08:29.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-mon[51517]: pgmap v226: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 321 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.9 MiB/s wr, 320 op/s; 170/1512 objects degraded (11.243%) 2024-09-17T00:08:29.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-mon[51517]: from='osd.5 [v2:172.21.15.194:6808/2761219763,v1:172.21.15.194:6809/2761219763]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:08:29.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-mon[51517]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:08:29.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:29 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 170/1512 objects degraded (11.243%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:29.936 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:29 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:29.936 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:29 smithi007 ceph-mon[65487]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-17T00:08:29.936 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:29 smithi007 ceph-mon[65487]: pgmap v226: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 321 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 2.9 MiB/s wr, 320 op/s; 170/1512 objects degraded (11.243%) 2024-09-17T00:08:29.936 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:29 smithi007 ceph-mon[65487]: from='osd.5 [v2:172.21.15.194:6808/2761219763,v1:172.21.15.194:6809/2761219763]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:08:29.936 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:29 smithi007 ceph-mon[65487]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-17T00:08:29.936 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:29 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 170/1512 objects degraded (11.243%), 23 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:30.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 219850 8.64 MB/sec execute 494 sec latency 249.647 ms 2024-09-17T00:08:30.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:08:30.339+0000 7ffbd9a90640 -1 osd.5 126 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:08:30.944 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:30 smithi007 ceph-mon[65487]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T00:08:30.944 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:30 smithi007 ceph-mon[65487]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T00:08:30.944 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:30 smithi007 ceph-mon[65487]: from='osd.5 [v2:172.21.15.194:6808/2761219763,v1:172.21.15.194:6809/2761219763]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:08:30.944 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:30 smithi007 ceph-mon[65487]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:08:30.944 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:30 smithi007 ceph-mon[65487]: from='osd.5 ' entity='osd.5' 2024-09-17T00:08:30.945 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:30 smithi007 ceph-mon[65487]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 323 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.8 MiB/s wr, 315 op/s; 175/1539 objects degraded (11.371%) 2024-09-17T00:08:31.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-mon[51517]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-17T00:08:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-mon[51517]: osdmap e129: 8 total, 7 up, 8 in 2024-09-17T00:08:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-mon[51517]: from='osd.5 [v2:172.21.15.194:6808/2761219763,v1:172.21.15.194:6809/2761219763]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:08:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-mon[51517]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:08:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-mon[51517]: from='osd.5 ' entity='osd.5' 2024-09-17T00:08:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:30 smithi194 ceph-mon[51517]: pgmap v228: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 323 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.8 MiB/s wr, 315 op/s; 175/1539 objects degraded (11.371%) 2024-09-17T00:08:31.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 220108 8.65 MB/sec execute 495 sec latency 63.060 ms 2024-09-17T00:08:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:31 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:08:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:31 smithi007 ceph-mon[65487]: osd.5 [v2:172.21.15.194:6808/2761219763,v1:172.21.15.194:6809/2761219763] boot 2024-09-17T00:08:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:31 smithi007 ceph-mon[65487]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T00:08:32.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:08:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:31 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:08:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:31 smithi194 ceph-mon[51517]: osd.5 [v2:172.21.15.194:6808/2761219763,v1:172.21.15.194:6809/2761219763] boot 2024-09-17T00:08:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:31 smithi194 ceph-mon[51517]: osdmap e130: 8 total, 8 up, 8 in 2024-09-17T00:08:32.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-17T00:08:32.313 INFO:teuthology.orchestra.run.smithi007.stdout: 5 220193 8.63 MB/sec execute 496 sec latency 949.923 ms 2024-09-17T00:08:33.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 220355 8.63 MB/sec execute 497 sec latency 1949.950 ms 2024-09-17T00:08:33.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:33 smithi007 ceph-mon[65487]: osdmap e131: 8 total, 8 up, 8 in 2024-09-17T00:08:33.466 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:33 smithi007 ceph-mon[65487]: pgmap v231: 97 pgs: 6 peering, 5 active+undersized, 20 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 323 op/s; 150/1656 objects degraded (9.058%) 2024-09-17T00:08:33.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:33 smithi194 ceph-mon[51517]: osdmap e131: 8 total, 8 up, 8 in 2024-09-17T00:08:33.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:33 smithi194 ceph-mon[51517]: pgmap v231: 97 pgs: 6 peering, 5 active+undersized, 20 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 323 op/s; 150/1656 objects degraded (9.058%) 2024-09-17T00:08:34.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 220528 8.61 MB/sec execute 498 sec latency 2032.538 ms 2024-09-17T00:08:34.504 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:34 smithi007 ceph-mon[65487]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T00:08:34.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:34 smithi194 ceph-mon[51517]: osdmap e132: 8 total, 8 up, 8 in 2024-09-17T00:08:35.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 220768 8.60 MB/sec execute 499 sec latency 81.694 ms 2024-09-17T00:08:35.562 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:35 smithi007 ceph-mon[65487]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T00:08:35.562 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:35 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 150/1656 objects degraded (9.058%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:35.562 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:35 smithi007 ceph-mon[65487]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T00:08:35.562 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:35 smithi007 ceph-mon[65487]: pgmap v235: 97 pgs: 6 peering, 5 active+undersized, 20 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.1 MiB/s wr, 64 op/s; 150/1656 objects degraded (9.058%) 2024-09-17T00:08:35.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:35 smithi194 ceph-mon[51517]: osdmap e133: 8 total, 8 up, 8 in 2024-09-17T00:08:35.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:35 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 150/1656 objects degraded (9.058%), 20 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:35.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:35 smithi194 ceph-mon[51517]: osdmap e134: 8 total, 8 up, 8 in 2024-09-17T00:08:35.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:35 smithi194 ceph-mon[51517]: pgmap v235: 97 pgs: 6 peering, 5 active+undersized, 20 active+undersized+degraded, 66 active+clean; 336 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.1 MiB/s wr, 64 op/s; 150/1656 objects degraded (9.058%) 2024-09-17T00:08:36.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 221050 8.59 MB/sec execute 500 sec latency 86.468 ms 2024-09-17T00:08:36.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:36 smithi007 ceph-mon[65487]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T00:08:37.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:36 smithi194 ceph-mon[51517]: osdmap e135: 8 total, 8 up, 8 in 2024-09-17T00:08:37.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 221334 8.58 MB/sec execute 501 sec latency 49.480 ms 2024-09-17T00:08:37.854 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:37 smithi007 ceph-mon[65487]: pgmap v237: 97 pgs: 5 peering, 92 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s wr, 288 op/s; 828 KiB/s, 35 objects/s recovering 2024-09-17T00:08:38.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:37 smithi194 ceph-mon[51517]: pgmap v237: 97 pgs: 5 peering, 92 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s wr, 288 op/s; 828 KiB/s, 35 objects/s recovering 2024-09-17T00:08:38.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 221630 8.59 MB/sec execute 502 sec latency 64.868 ms 2024-09-17T00:08:38.903 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:38 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 150/1656 objects degraded (9.058%), 20 pgs degraded) 2024-09-17T00:08:38.903 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:38 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:08:39.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:38 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 150/1656 objects degraded (9.058%), 20 pgs degraded) 2024-09-17T00:08:39.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:38 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:08:39.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 222021 8.59 MB/sec execute 503 sec latency 40.383 ms 2024-09-17T00:08:39.990 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:39 smithi007 ceph-mon[65487]: pgmap v238: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.4 MiB/s wr, 235 op/s; 785 KiB/s, 34 objects/s recovering 2024-09-17T00:08:39.990 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:39.990 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:40.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:39 smithi194 ceph-mon[51517]: pgmap v238: 97 pgs: 97 active+clean; 335 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 2.4 MiB/s wr, 235 op/s; 785 KiB/s, 34 objects/s recovering 2024-09-17T00:08:40.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:40.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:40.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 222333 8.58 MB/sec execute 504 sec latency 72.520 ms 2024-09-17T00:08:41.022 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:40 smithi007 ceph-mon[65487]: pgmap v239: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.4 MiB/s wr, 491 op/s; 664 KiB/s, 29 objects/s recovering 2024-09-17T00:08:41.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:40 smithi194 ceph-mon[51517]: pgmap v239: 97 pgs: 97 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.4 MiB/s wr, 491 op/s; 664 KiB/s, 29 objects/s recovering 2024-09-17T00:08:41.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 222626 8.58 MB/sec execute 505 sec latency 72.564 ms 2024-09-17T00:08:42.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 222799 8.58 MB/sec execute 506 sec latency 107.667 ms 2024-09-17T00:08:43.061 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:42 smithi007 ceph-mon[65487]: pgmap v240: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.1 MiB/s wr, 474 op/s; 524 KiB/s, 23 objects/s recovering 2024-09-17T00:08:43.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:42 smithi194 ceph-mon[51517]: pgmap v240: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.2 KiB/s rd, 4.1 MiB/s wr, 474 op/s; 524 KiB/s, 23 objects/s recovering 2024-09-17T00:08:43.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 222965 8.58 MB/sec execute 507 sec latency 91.283 ms 2024-09-17T00:08:44.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:43 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:44.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:43 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:44.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:43 smithi007 ceph-mon[65487]: Upgrade: osd.6 is safe to restart 2024-09-17T00:08:44.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:43 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:44.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:43 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-17T00:08:44.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:43 smithi194 ceph-mon[51517]: Upgrade: osd.6 is safe to restart 2024-09-17T00:08:44.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 223292 8.58 MB/sec execute 508 sec latency 120.985 ms 2024-09-17T00:08:45.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 223574 8.59 MB/sec execute 509 sec latency 71.030 ms 2024-09-17T00:08:45.344 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: Upgrade: Updating osd.6 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: Deploying daemon osd.6 on smithi194 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: pgmap v241: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 MiB/s wr, 383 op/s; 424 KiB/s, 18 objects/s recovering 2024-09-17T00:08:45.652 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: Upgrade: Updating osd.6 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: Deploying daemon osd.6 on smithi194 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: pgmap v241: 97 pgs: 97 active+clean; 329 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 MiB/s wr, 383 op/s; 424 KiB/s, 18 objects/s recovering 2024-09-17T00:08:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:46.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 223845 8.59 MB/sec execute 510 sec latency 50.081 ms 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (8m) 99s ago 17m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (4m) 99s ago 16m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (4m) 23s ago 16m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (7m) 99s ago 16m 53.4M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (11m) 99s ago 11m 37.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (11m) 23s ago 11m 15.3M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:08:46.379 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (7m) 99s ago 18m 503M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (6m) 23s ago 16m 442M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (6m) 99s ago 18m 50.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (5m) 23s ago 16m 40.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (10m) 99s ago 10m 270M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (10m) 23s ago 10m 88.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (7m) 99s ago 16m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 23s ago 16m 16.2M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (4m) 99s ago 15m 321M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (3m) 99s ago 15m 258M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (2m) 99s ago 14m 215M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (108s) 99s ago 14m 11.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (65s) 23s ago 13m 234M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:08:46.380 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (27s) 23s ago 13m 11.1M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:08:46.381 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (12m) 23s ago 12m 446M 2704M 17.2.0 e1d6a67b021e 5663e879c4b9 2024-09-17T00:08:46.381 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (12m) 23s ago 12m 640M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:08:46.381 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (6m) 99s ago 15m 66.3M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:08:46.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:46 smithi007 ceph-mon[65487]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:46.809 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:46 smithi194 ceph-mon[51517]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:46.965 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:08:46.965 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:08:46.965 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:08:46.965 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:46.965 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:08:46.965 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 6 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 10 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:08:46.966 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:08:47.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 224023 8.58 MB/sec execute 511 sec latency 80.508 ms 2024-09-17T00:08:47.465 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:08:47.465 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:08:47.465 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "12/23 daemons upgraded", 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:08:47.466 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:08:47.706 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:47 smithi007 ceph-mon[65487]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:47.706 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:47 smithi007 ceph-mon[65487]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:47.706 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:47 smithi007 ceph-mon[65487]: pgmap v242: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 158 KiB/s, 6 objects/s recovering 2024-09-17T00:08:47.706 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:47 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3739128450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:47.763 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:47 smithi194 ceph-mon[51517]: from='client.34390 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:47.763 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:47 smithi194 ceph-mon[51517]: from='client.34394 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:47.763 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:47 smithi194 ceph-mon[51517]: pgmap v242: 97 pgs: 97 active+clean; 332 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.2 MiB/s wr, 424 op/s; 158 KiB/s, 6 objects/s recovering 2024-09-17T00:08:47.764 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:47 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3739128450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:08:48.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:47 smithi194 systemd[1]: Stopping Ceph osd.6 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:08:48.213 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:08:48.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 224274 8.57 MB/sec execute 512 sec latency 80.210 ms 2024-09-17T00:08:48.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:48 smithi194 ceph-mon[51517]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:48.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:48 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2310723535' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:08:48.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:48 smithi194 ceph-mon[51517]: osd.6 marked itself down and dead 2024-09-17T00:08:48.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[35912]: 2024-09-17T00:08:48.305+0000 7fdf40f9c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:08:48.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[35912]: 2024-09-17T00:08:48.305+0000 7fdf40f9c700 -1 osd.6 135 *** Got signal Terminated *** 2024-09-17T00:08:48.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[35912]: 2024-09-17T00:08:48.305+0000 7fdf40f9c700 -1 osd.6 135 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:08:48.734 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:48 smithi007 ceph-mon[65487]: from='client.34402 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:08:48.734 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:48 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2310723535' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:08:48.734 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:48 smithi007 ceph-mon[65487]: osd.6 marked itself down and dead 2024-09-17T00:08:49.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 224495 8.56 MB/sec execute 513 sec latency 799.615 ms 2024-09-17T00:08:49.393 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:49 smithi194 podman[60448]: 2024-09-17 00:08:49.138707461 +0000 UTC m=+0.977039888 container died 5663e879c4b9bb473901761ac1ba438823aadae69c1939aa23c023f3b478c6a6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, io.openshift.expose-services=, vcs-type=git, RELEASE=HEAD, architecture=x86_64, distribution-scope=public, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-17T00:08:49.546 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:49 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:08:49.546 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:49 smithi007 ceph-mon[65487]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T00:08:49.546 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:49 smithi007 ceph-mon[65487]: pgmap v244: 97 pgs: 10 stale+active+clean, 87 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-17T00:08:49.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:49 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:08:49.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:49 smithi194 ceph-mon[51517]: osdmap e136: 8 total, 7 up, 8 in 2024-09-17T00:08:49.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:49 smithi194 ceph-mon[51517]: pgmap v244: 97 pgs: 10 stale+active+clean, 87 active+clean; 333 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.3 MiB/s wr, 428 op/s 2024-09-17T00:08:49.981 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:49 smithi194 podman[60448]: 2024-09-17 00:08:49.681591164 +0000 UTC m=+1.519923589 container cleanup 5663e879c4b9bb473901761ac1ba438823aadae69c1939aa23c023f3b478c6a6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, ceph=True, io.openshift.tags=base centos centos-stream) 2024-09-17T00:08:49.981 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:49 smithi194 bash[60448]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6 2024-09-17T00:08:50.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 224682 8.54 MB/sec execute 514 sec latency 1042.907 ms 2024-09-17T00:08:50.395 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:49 smithi194 podman[60473]: 2024-09-17 00:08:49.983425883 +0000 UTC m=+0.840355168 container remove 5663e879c4b9bb473901761ac1ba438823aadae69c1939aa23c023f3b478c6a6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vendor=Red Hat, Inc., architecture=x86_64, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, vcs-type=git, version=8, release=754, maintainer=Guillaume Abrioux ) 2024-09-17T00:08:50.559 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:50 smithi007 ceph-mon[65487]: osdmap e137: 8 total, 7 up, 8 in 2024-09-17T00:08:50.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:50 smithi194 ceph-mon[51517]: osdmap e137: 8 total, 7 up, 8 in 2024-09-17T00:08:50.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:50 smithi194 podman[60563]: 2024-09-17 00:08:50.301630843 +0000 UTC m=+0.020064846 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:08:50.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:50 smithi194 podman[60563]: 2024-09-17 00:08:50.417893447 +0000 UTC m=+0.136327431 container create 97db3395127b895c908ac925e3996cefc4e152b21628d145da07190d724fd3bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-17T00:08:51.121 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:50 smithi194 podman[60563]: 2024-09-17 00:08:50.888062978 +0000 UTC m=+0.606496960 container init 97db3395127b895c908ac925e3996cefc4e152b21628d145da07190d724fd3bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-17T00:08:51.121 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:50 smithi194 podman[60563]: 2024-09-17 00:08:50.893047556 +0000 UTC m=+0.611481531 container start 97db3395127b895c908ac925e3996cefc4e152b21628d145da07190d724fd3bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=squid-c2ddf97, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T00:08:51.122 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:50 smithi194 podman[60563]: 2024-09-17 00:08:50.987726873 +0000 UTC m=+0.706160861 container attach 97db3395127b895c908ac925e3996cefc4e152b21628d145da07190d724fd3bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T00:08:51.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 224886 8.54 MB/sec execute 515 sec latency 40.400 ms 2024-09-17T00:08:51.408 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:51 smithi194 podman[60585]: 2024-09-17 00:08:51.154641377 +0000 UTC m=+0.021418862 container died 97db3395127b895c908ac925e3996cefc4e152b21628d145da07190d724fd3bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-17T00:08:51.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:51 smithi007 ceph-mon[65487]: pgmap v246: 97 pgs: 3 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 79 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.8 MiB/s wr, 369 op/s; 64/1662 objects degraded (3.851%) 2024-09-17T00:08:51.853 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:51 smithi194 ceph-mon[51517]: pgmap v246: 97 pgs: 3 active+undersized, 7 stale+active+clean, 8 active+undersized+degraded, 79 active+clean; 334 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.8 MiB/s wr, 369 op/s; 64/1662 objects degraded (3.851%) 2024-09-17T00:08:52.123 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:51 smithi194 podman[60585]: 2024-09-17 00:08:51.856650329 +0000 UTC m=+0.723427817 container remove 97db3395127b895c908ac925e3996cefc4e152b21628d145da07190d724fd3bf (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T00:08:52.123 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:51 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-17T00:08:52.123 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:51 smithi194 systemd[1]: Stopped Ceph osd.6 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:08:52.123 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:51 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.6.service: Consumed 3min 51.772s CPU time. 2024-09-17T00:08:52.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 225145 8.54 MB/sec execute 516 sec latency 42.270 ms 2024-09-17T00:08:52.397 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:52 smithi194 systemd[1]: Starting Ceph osd.6 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:08:52.398 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:52 smithi194 podman[60682]: 2024-09-17 00:08:52.266933021 +0000 UTC m=+0.020897880 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:08:52.625 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:52 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 64/1662 objects degraded (3.851%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:52.659 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:52 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 64/1662 objects degraded (3.851%), 8 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:52.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:52 smithi194 podman[60682]: 2024-09-17 00:08:52.400630172 +0000 UTC m=+0.154595019 container create bdda511eb68bf5d1d018651505b3c9ed3690813d68856dcddf43f5249129dedd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:08:53.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 225465 8.54 MB/sec execute 517 sec latency 98.457 ms 2024-09-17T00:08:53.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:52 smithi194 podman[60682]: 2024-09-17 00:08:52.914493101 +0000 UTC m=+0.668457942 container init bdda511eb68bf5d1d018651505b3c9ed3690813d68856dcddf43f5249129dedd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T00:08:53.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:52 smithi194 podman[60682]: 2024-09-17 00:08:52.918595902 +0000 UTC m=+0.672560746 container start bdda511eb68bf5d1d018651505b3c9ed3690813d68856dcddf43f5249129dedd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-17T00:08:53.410 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 podman[60682]: 2024-09-17 00:08:53.014491612 +0000 UTC m=+0.768456465 container attach bdda511eb68bf5d1d018651505b3c9ed3690813d68856dcddf43f5249129dedd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True) 2024-09-17T00:08:53.410 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:53.410 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 bash[60682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:53.410 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:53.410 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 bash[60682]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:08:53.696 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:53 smithi007 ceph-mon[65487]: pgmap v247: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 424 op/s; 180/1551 objects degraded (11.605%) 2024-09-17T00:08:53.886 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:53 smithi194 ceph-mon[51517]: pgmap v247: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.7 MiB/s wr, 424 op/s; 180/1551 objects degraded (11.605%) 2024-09-17T00:08:54.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T00:08:54.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 bash[60682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T00:08:54.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T00:08:54.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:53 smithi194 bash[60682]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-17T00:08:54.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 225843 8.53 MB/sec execute 518 sec latency 74.447 ms 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 bash[60682]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 bash[60682]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 bash[60682]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 bash[60682]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-17T00:08:54.559 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate[60695]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T00:08:54.560 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 bash[60682]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-17T00:08:54.560 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 podman[60682]: 2024-09-17 00:08:54.207594075 +0000 UTC m=+1.961558930 container died bdda511eb68bf5d1d018651505b3c9ed3690813d68856dcddf43f5249129dedd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T00:08:55.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:54 smithi194 podman[60820]: 2024-09-17 00:08:54.878309293 +0000 UTC m=+0.664750102 container remove bdda511eb68bf5d1d018651505b3c9ed3690813d68856dcddf43f5249129dedd (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-17T00:08:55.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 podman[60876]: 2024-09-17 00:08:54.993207431 +0000 UTC m=+0.017807808 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:08:55.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 podman[60876]: 2024-09-17 00:08:55.095988181 +0000 UTC m=+0.120588539 container create 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-17T00:08:55.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 226091 8.53 MB/sec execute 519 sec latency 161.315 ms 2024-09-17T00:08:55.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 podman[60876]: 2024-09-17 00:08:55.342000633 +0000 UTC m=+0.366601012 container init 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T00:08:55.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 podman[60876]: 2024-09-17 00:08:55.345979159 +0000 UTC m=+0.370579527 container start 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-17T00:08:55.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-osd[60895]: -- 172.21.15.194:0/818898987 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bf216b81e0 con 0x55bf2168e000 2024-09-17T00:08:55.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 bash[60876]: 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe 2024-09-17T00:08:55.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:55 smithi194 systemd[1]: Started Ceph osd.6 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:08:55.770 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:55.770 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:55.770 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:55 smithi007 ceph-mon[65487]: pgmap v248: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.4 MiB/s wr, 227 op/s; 180/1551 objects degraded (11.605%) 2024-09-17T00:08:55.770 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:55.770 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:55.770 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:55 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:08:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:08:56.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-mon[51517]: pgmap v248: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 328 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 2.4 MiB/s wr, 227 op/s; 180/1551 objects degraded (11.605%) 2024-09-17T00:08:56.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:56.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:08:56.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:55 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:08:56.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 226310 8.53 MB/sec execute 520 sec latency 52.741 ms 2024-09-17T00:08:57.050 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:56 smithi007 ceph-mon[65487]: pgmap v249: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 323 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 453 op/s; 172/1545 objects degraded (11.133%) 2024-09-17T00:08:57.050 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:56 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 172/1545 objects degraded (11.133%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:56 smithi194 ceph-mon[51517]: pgmap v249: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 323 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.7 MiB/s wr, 453 op/s; 172/1545 objects degraded (11.133%) 2024-09-17T00:08:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:56 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 172/1545 objects degraded (11.133%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T00:08:57.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 226631 8.54 MB/sec execute 521 sec latency 113.015 ms 2024-09-17T00:08:57.659 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:08:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:08:57.235+0000 7f0258677640 -1 Falling back to public interface 2024-09-17T00:08:58.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 226940 8.54 MB/sec execute 522 sec latency 65.001 ms 2024-09-17T00:08:59.001 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:08:58 smithi194 ceph-mon[51517]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 392 op/s; 171/1569 objects degraded (10.899%) 2024-09-17T00:08:59.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:08:58 smithi007 ceph-mon[65487]: pgmap v250: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 324 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 3.4 MiB/s wr, 392 op/s; 171/1569 objects degraded (10.899%) 2024-09-17T00:08:59.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 227282 8.55 MB/sec execute 523 sec latency 46.805 ms 2024-09-17T00:09:00.314 INFO:teuthology.orchestra.run.smithi007.stdout: 5 227469 8.55 MB/sec execute 524 sec latency 85.893 ms 2024-09-17T00:09:00.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.391 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:00 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:00 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:00 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:00.521 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:00 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:01.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 227671 8.54 MB/sec execute 525 sec latency 42.987 ms 2024-09-17T00:09:01.425 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:01 smithi007 ceph-mon[65487]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 322 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 364 op/s; 177/1596 objects degraded (11.090%) 2024-09-17T00:09:01.589 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:01 smithi194 ceph-mon[51517]: pgmap v251: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 322 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 364 op/s; 177/1596 objects degraded (11.090%) 2024-09-17T00:09:02.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 228010 8.53 MB/sec execute 526 sec latency 60.611 ms 2024-09-17T00:09:02.951 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:02 smithi007 ceph-mon[65487]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.4 MiB/s wr, 404 op/s; 186/1677 objects degraded (11.091%) 2024-09-17T00:09:02.951 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:02 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 186/1677 objects degraded (11.091%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:03.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:02 smithi194 ceph-mon[51517]: pgmap v252: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.4 MiB/s wr, 404 op/s; 186/1677 objects degraded (11.091%) 2024-09-17T00:09:03.125 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:02 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 186/1677 objects degraded (11.091%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:03.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 228327 8.52 MB/sec execute 527 sec latency 46.407 ms 2024-09-17T00:09:03.909 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:09:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:09:03.651+0000 7f0258677640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-17T00:09:04.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 228561 8.52 MB/sec execute 528 sec latency 64.478 ms 2024-09-17T00:09:04.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:04.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:04.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:04.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:09:04.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:04.790 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:04.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:04.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:04.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:04.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:04.791 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:04 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:05.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:09:04.718+0000 7f0258677640 -1 osd.6 135 log_to_monitors true 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:05.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:05.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:05.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:05.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:04 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:05.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 228951 8.53 MB/sec execute 529 sec latency 53.855 ms 2024-09-17T00:09:05.799 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:05 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:05.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:05 smithi007 ceph-mon[65487]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-17T00:09:05.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:05 smithi007 ceph-mon[65487]: from='osd.6 [v2:172.21.15.194:6816/1252501579,v1:172.21.15.194:6817/1252501579]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:09:05.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:05 smithi007 ceph-mon[65487]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:09:05.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:05 smithi007 ceph-mon[65487]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.8 MiB/s wr, 367 op/s; 186/1677 objects degraded (11.091%) 2024-09-17T00:09:06.158 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:09:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:09:05.715+0000 7f0250422640 -1 osd.6 135 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:09:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:05 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:05 smithi194 ceph-mon[51517]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-17T00:09:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:05 smithi194 ceph-mon[51517]: from='osd.6 [v2:172.21.15.194:6816/1252501579,v1:172.21.15.194:6817/1252501579]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:09:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:05 smithi194 ceph-mon[51517]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-17T00:09:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:05 smithi194 ceph-mon[51517]: pgmap v253: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 335 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.8 MiB/s wr, 367 op/s; 186/1677 objects degraded (11.091%) 2024-09-17T00:09:06.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 229431 8.52 MB/sec execute 530 sec latency 42.925 ms 2024-09-17T00:09:06.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:06 smithi007 ceph-mon[65487]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T00:09:06.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:06 smithi007 ceph-mon[65487]: osdmap e138: 8 total, 7 up, 8 in 2024-09-17T00:09:06.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:06 smithi007 ceph-mon[65487]: from='osd.6 [v2:172.21.15.194:6816/1252501579,v1:172.21.15.194:6817/1252501579]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:06.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:06 smithi007 ceph-mon[65487]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:06 smithi194 ceph-mon[51517]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-17T00:09:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:06 smithi194 ceph-mon[51517]: osdmap e138: 8 total, 7 up, 8 in 2024-09-17T00:09:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:06 smithi194 ceph-mon[51517]: from='osd.6 [v2:172.21.15.194:6816/1252501579,v1:172.21.15.194:6817/1252501579]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:07.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:06 smithi194 ceph-mon[51517]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:07.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 229568 8.51 MB/sec execute 531 sec latency 623.843 ms 2024-09-17T00:09:07.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:07 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:09:07.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:07 smithi007 ceph-mon[65487]: osd.6 [v2:172.21.15.194:6816/1252501579,v1:172.21.15.194:6817/1252501579] boot 2024-09-17T00:09:07.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:07 smithi007 ceph-mon[65487]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T00:09:07.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:07 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:09:07.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:07 smithi007 ceph-mon[65487]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.9 MiB/s wr, 521 op/s; 187/1587 objects degraded (11.783%) 2024-09-17T00:09:07.976 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:07 smithi007 ceph-mon[65487]: osdmap e140: 8 total, 8 up, 8 in 2024-09-17T00:09:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:07 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:09:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:07 smithi194 ceph-mon[51517]: osd.6 [v2:172.21.15.194:6816/1252501579,v1:172.21.15.194:6817/1252501579] boot 2024-09-17T00:09:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:07 smithi194 ceph-mon[51517]: osdmap e139: 8 total, 8 up, 8 in 2024-09-17T00:09:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:07 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-17T00:09:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:07 smithi194 ceph-mon[51517]: pgmap v256: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.9 KiB/s rd, 4.9 MiB/s wr, 521 op/s; 187/1587 objects degraded (11.783%) 2024-09-17T00:09:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:07 smithi194 ceph-mon[51517]: osdmap e140: 8 total, 8 up, 8 in 2024-09-17T00:09:08.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 229673 8.50 MB/sec execute 532 sec latency 1120.238 ms 2024-09-17T00:09:09.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 229840 8.50 MB/sec execute 533 sec latency 963.728 ms 2024-09-17T00:09:09.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:09 smithi007 ceph-mon[65487]: osdmap e141: 8 total, 8 up, 8 in 2024-09-17T00:09:09.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:09 smithi007 ceph-mon[65487]: pgmap v259: 97 pgs: 1 peering, 10 active+undersized, 24 active+undersized+degraded, 62 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 4.3 MiB/s wr, 549 op/s; 181/1566 objects degraded (11.558%) 2024-09-17T00:09:09.777 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:09 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 187/1587 objects degraded (11.783%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:10.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:09 smithi194 ceph-mon[51517]: osdmap e141: 8 total, 8 up, 8 in 2024-09-17T00:09:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:09 smithi194 ceph-mon[51517]: pgmap v259: 97 pgs: 1 peering, 10 active+undersized, 24 active+undersized+degraded, 62 active+clean; 320 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 4.3 MiB/s wr, 549 op/s; 181/1566 objects degraded (11.558%) 2024-09-17T00:09:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:09 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 187/1587 objects degraded (11.783%), 25 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:10.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 230084 8.50 MB/sec execute 534 sec latency 447.431 ms 2024-09-17T00:09:10.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:10 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:10.785 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:10 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:11.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:10 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:11.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:10 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:11.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 230248 8.50 MB/sec execute 535 sec latency 263.975 ms 2024-09-17T00:09:11.861 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:11 smithi007 ceph-mon[65487]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T00:09:11.861 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:11 smithi007 ceph-mon[65487]: pgmap v261: 97 pgs: 1 active+undersized+remapped, 1 peering, 7 active+undersized, 10 active+undersized+degraded, 78 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 2.5 MiB/s wr, 305 op/s; 84/1545 objects degraded (5.437%); 1.2 MiB/s, 64 keys/s, 22 objects/s recovering 2024-09-17T00:09:11.861 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:11 smithi007 ceph-mon[65487]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T00:09:12.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:11 smithi194 ceph-mon[51517]: osdmap e142: 8 total, 8 up, 8 in 2024-09-17T00:09:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:11 smithi194 ceph-mon[51517]: pgmap v261: 97 pgs: 1 active+undersized+remapped, 1 peering, 7 active+undersized, 10 active+undersized+degraded, 78 active+clean; 321 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 2.5 MiB/s wr, 305 op/s; 84/1545 objects degraded (5.437%); 1.2 MiB/s, 64 keys/s, 22 objects/s recovering 2024-09-17T00:09:12.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:11 smithi194 ceph-mon[51517]: osdmap e143: 8 total, 8 up, 8 in 2024-09-17T00:09:12.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 230478 8.50 MB/sec execute 536 sec latency 692.821 ms 2024-09-17T00:09:13.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 230627 8.49 MB/sec execute 537 sec latency 666.472 ms 2024-09-17T00:09:13.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:13 smithi194 ceph-mon[51517]: osdmap e144: 8 total, 8 up, 8 in 2024-09-17T00:09:13.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:13 smithi194 ceph-mon[51517]: pgmap v264: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 92 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.7 MiB/s wr, 344 op/s; 30/1551 objects degraded (1.934%); 2.2 MiB/s, 64 keys/s, 42 objects/s recovering 2024-09-17T00:09:13.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:13 smithi007 ceph-mon[65487]: osdmap e144: 8 total, 8 up, 8 in 2024-09-17T00:09:13.953 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:13 smithi007 ceph-mon[65487]: pgmap v264: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 92 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.7 MiB/s wr, 344 op/s; 30/1551 objects degraded (1.934%); 2.2 MiB/s, 64 keys/s, 42 objects/s recovering 2024-09-17T00:09:14.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 230874 8.49 MB/sec execute 538 sec latency 279.715 ms 2024-09-17T00:09:14.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:14 smithi007 ceph-mon[65487]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T00:09:14.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:14 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 30/1551 objects degraded (1.934%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:14 smithi194 ceph-mon[51517]: osdmap e145: 8 total, 8 up, 8 in 2024-09-17T00:09:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:14 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 30/1551 objects degraded (1.934%), 3 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:15.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 231103 8.49 MB/sec execute 539 sec latency 54.251 ms 2024-09-17T00:09:16.037 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:15 smithi007 ceph-mon[65487]: pgmap v266: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 92 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.2 MiB/s wr, 85 op/s; 30/1551 objects degraded (1.934%); 1.0 MiB/s, 19 objects/s recovering 2024-09-17T00:09:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:15 smithi194 ceph-mon[51517]: pgmap v266: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 92 active+clean; 324 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.2 MiB/s wr, 85 op/s; 30/1551 objects degraded (1.934%); 1.0 MiB/s, 19 objects/s recovering 2024-09-17T00:09:16.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 231278 8.48 MB/sec execute 540 sec latency 82.969 ms 2024-09-17T00:09:17.051 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:16 smithi007 ceph-mon[65487]: pgmap v267: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.8 MiB/s wr, 276 op/s; 872 KiB/s, 21 objects/s recovering 2024-09-17T00:09:17.051 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:16 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1551 objects degraded (1.934%), 3 pgs degraded) 2024-09-17T00:09:17.051 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:16 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:09:17.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:16 smithi194 ceph-mon[51517]: pgmap v267: 97 pgs: 97 active+clean; 326 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.3 KiB/s rd, 3.8 MiB/s wr, 276 op/s; 872 KiB/s, 21 objects/s recovering 2024-09-17T00:09:17.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:16 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/1551 objects degraded (1.934%), 3 pgs degraded) 2024-09-17T00:09:17.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:16 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:09:17.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 231472 8.47 MB/sec execute 541 sec latency 74.106 ms 2024-09-17T00:09:18.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 231682 8.46 MB/sec execute 542 sec latency 77.197 ms 2024-09-17T00:09:18.676 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:09:19.120 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:18 smithi007 ceph-mon[65487]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:19.120 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:18 smithi007 ceph-mon[65487]: pgmap v268: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.0 MiB/s wr, 221 op/s; 142 KiB/s, 6 objects/s recovering 2024-09-17T00:09:19.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:18 smithi194 ceph-mon[51517]: from='client.34414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:19.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:18 smithi194 ceph-mon[51517]: pgmap v268: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 3.0 MiB/s wr, 221 op/s; 142 KiB/s, 6 objects/s recovering 2024-09-17T00:09:19.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 231918 8.45 MB/sec execute 543 sec latency 76.296 ms 2024-09-17T00:09:19.544 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (8m) 2m ago 17m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (5m) 2m ago 17m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (5m) 20s ago 17m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (8m) 2m ago 17m 53.4M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (11m) 2m ago 11m 37.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (11m) 20s ago 11m 15.4M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (7m) 2m ago 18m 503M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (6m) 20s ago 16m 442M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (6m) 2m ago 18m 50.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (6m) 20s ago 16m 40.9M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (11m) 2m ago 11m 270M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (11m) 20s ago 11m 88.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:09:19.545 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (8m) 2m ago 17m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (7m) 20s ago 16m 16.5M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (4m) 2m ago 16m 321M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (3m) 2m ago 15m 258M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (3m) 2m ago 15m 215M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (2m) 2m ago 14m 11.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (98s) 20s ago 14m 282M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (61s) 20s ago 13m 239M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (24s) 20s ago 13m 11.4M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 81dd179cd962 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (13m) 20s ago 13m 685M 2704M 17.2.0 e1d6a67b021e da7d66361cf2 2024-09-17T00:09:19.546 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (6m) 2m ago 16m 66.3M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:09:19.911 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:19 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:19.911 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:19 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:19.911 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:19 smithi007 ceph-mon[65487]: Upgrade: osd.7 is safe to restart 2024-09-17T00:09:19.911 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:19 smithi007 ceph-mon[65487]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:19.911 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:19 smithi007 ceph-mon[65487]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-17T00:09:20.101 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 7 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 11 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:09:20.102 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:09:20.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:19 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:20.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:19 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-17T00:09:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:19 smithi194 ceph-mon[51517]: Upgrade: osd.7 is safe to restart 2024-09-17T00:09:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:19 smithi194 ceph-mon[51517]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:20.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:19 smithi194 ceph-mon[51517]: from='client.34420 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:20.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 232096 8.44 MB/sec execute 544 sec latency 87.447 ms 2024-09-17T00:09:20.549 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "13/23 daemons upgraded", 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading osd daemons", 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:09:20.550 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/323117081' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: Upgrade: Updating osd.7 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: Deploying daemon osd.7 on smithi194 2024-09-17T00:09:20.930 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:20 smithi007 ceph-mon[65487]: pgmap v269: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.5 MiB/s wr, 220 op/s; 125 KiB/s, 5 objects/s recovering 2024-09-17T00:09:21.112 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/323117081' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: Upgrade: Updating osd.7 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: Deploying daemon osd.7 on smithi194 2024-09-17T00:09:21.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:20 smithi194 ceph-mon[51517]: pgmap v269: 97 pgs: 97 active+clean; 322 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.5 MiB/s wr, 220 op/s; 125 KiB/s, 5 objects/s recovering 2024-09-17T00:09:21.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 232387 8.45 MB/sec execute 545 sec latency 50.167 ms 2024-09-17T00:09:21.952 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:21 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3986999572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:09:22.078 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:21 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3986999572' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:09:22.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 232787 8.45 MB/sec execute 546 sec latency 46.057 ms 2024-09-17T00:09:22.989 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:22 smithi007 ceph-mon[65487]: pgmap v270: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.2 MiB/s wr, 336 op/s; 100 KiB/s, 4 objects/s recovering 2024-09-17T00:09:23.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 233035 8.45 MB/sec execute 547 sec latency 76.601 ms 2024-09-17T00:09:23.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:22 smithi194 ceph-mon[51517]: pgmap v270: 97 pgs: 97 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.2 MiB/s wr, 336 op/s; 100 KiB/s, 4 objects/s recovering 2024-09-17T00:09:24.108 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:23 smithi194 systemd[1]: Stopping Ceph osd.7 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:09:24.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 233304 8.44 MB/sec execute 548 sec latency 144.912 ms 2024-09-17T00:09:24.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:24 smithi007 ceph-mon[65487]: osd.7 marked itself down and dead 2024-09-17T00:09:24.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:24 smithi194 ceph-mon[51517]: osd.7 marked itself down and dead 2024-09-17T00:09:24.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:24 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[38881]: 2024-09-17T00:09:24.139+0000 7f9fc3ff5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:09:24.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:24 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[38881]: 2024-09-17T00:09:24.139+0000 7f9fc3ff5700 -1 osd.7 145 *** Got signal Terminated *** 2024-09-17T00:09:24.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:24 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[38881]: 2024-09-17T00:09:24.139+0000 7f9fc3ff5700 -1 osd.7 145 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:09:25.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:24 smithi194 podman[62679]: 2024-09-17 00:09:24.902034522 +0000 UTC m=+0.877547145 container died da7d66361cf2a263701e41b0b9fde7474a1a26c05c66892ae8989c02d952fbf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., ceph=True, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.openshift.expose-services=, RELEASE=HEAD, release=754, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux ) 2024-09-17T00:09:25.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 233320 8.43 MB/sec execute 549 sec latency 1076.811 ms 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: pgmap v272: 97 pgs: 12 stale+active+clean, 85 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.2 MiB/s wr, 336 op/s; 100 KiB/s, 4 objects/s recovering 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:25.487 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:25 smithi007 ceph-mon[65487]: osdmap e147: 8 total, 7 up, 8 in 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: osdmap e146: 8 total, 7 up, 8 in 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: pgmap v272: 97 pgs: 12 stale+active+clean, 85 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 3.2 MiB/s wr, 336 op/s; 100 KiB/s, 4 objects/s recovering 2024-09-17T00:09:25.563 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:25.564 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:25 smithi194 ceph-mon[51517]: osdmap e147: 8 total, 7 up, 8 in 2024-09-17T00:09:25.564 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:25 smithi194 podman[62679]: 2024-09-17 00:09:25.297328659 +0000 UTC m=+1.272841276 container cleanup da7d66361cf2a263701e41b0b9fde7474a1a26c05c66892ae8989c02d952fbf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, version=8, ceph=True, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.expose-services=, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, distribution-scope=public, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T00:09:25.564 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:25 smithi194 bash[62679]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7 2024-09-17T00:09:25.891 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:25 smithi194 podman[62706]: 2024-09-17 00:09:25.565679341 +0000 UTC m=+0.656382834 container remove da7d66361cf2a263701e41b0b9fde7474a1a26c05c66892ae8989c02d952fbf6 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, distribution-scope=public, com.redhat.component=centos-stream-container, version=8, release=754, name=centos-stream, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, ceph=True, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-17T00:09:26.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:25 smithi194 podman[62791]: 2024-09-17 00:09:25.797538631 +0000 UTC m=+0.018984179 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:09:26.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:25 smithi194 podman[62791]: 2024-09-17 00:09:25.933436733 +0000 UTC m=+0.154882265 container create e6f5148573523acd49e2a205faf93c7686162f5d39d46a81263758a302c8cd16 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T00:09:26.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 233592 8.42 MB/sec execute 550 sec latency 49.825 ms 2024-09-17T00:09:26.659 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:26 smithi194 podman[62791]: 2024-09-17 00:09:26.169836582 +0000 UTC m=+0.391282120 container init e6f5148573523acd49e2a205faf93c7686162f5d39d46a81263758a302c8cd16 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T00:09:26.659 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:26 smithi194 podman[62791]: 2024-09-17 00:09:26.173416144 +0000 UTC m=+0.394861680 container start e6f5148573523acd49e2a205faf93c7686162f5d39d46a81263758a302c8cd16 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-17T00:09:26.659 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:26 smithi194 podman[62791]: 2024-09-17 00:09:26.269999837 +0000 UTC m=+0.491445372 container attach e6f5148573523acd49e2a205faf93c7686162f5d39d46a81263758a302c8cd16 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T00:09:26.659 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:26 smithi194 podman[62791]: 2024-09-17 00:09:26.374610747 +0000 UTC m=+0.596056290 container died e6f5148573523acd49e2a205faf93c7686162f5d39d46a81263758a302c8cd16 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T00:09:27.004 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:26 smithi007 ceph-mon[65487]: pgmap v274: 97 pgs: 2 active+undersized, 8 peering, 3 stale+active+clean, 22 active+undersized+degraded, 62 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.5 MiB/s wr, 417 op/s; 138/1533 objects degraded (9.002%) 2024-09-17T00:09:27.004 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:26 smithi007 ceph-mon[65487]: Health check failed: Degraded data redundancy: 138/1533 objects degraded (9.002%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:27.041 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:26 smithi194 ceph-mon[51517]: pgmap v274: 97 pgs: 2 active+undersized, 8 peering, 3 stale+active+clean, 22 active+undersized+degraded, 62 active+clean; 322 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.5 MiB/s wr, 417 op/s; 138/1533 objects degraded (9.002%) 2024-09-17T00:09:27.041 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:26 smithi194 ceph-mon[51517]: Health check failed: Degraded data redundancy: 138/1533 objects degraded (9.002%), 22 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:27.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 234054 8.44 MB/sec execute 551 sec latency 75.484 ms 2024-09-17T00:09:27.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 podman[62819]: 2024-09-17 00:09:27.04344956 +0000 UTC m=+0.662845471 container remove e6f5148573523acd49e2a205faf93c7686162f5d39d46a81263758a302c8cd16 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-17T00:09:27.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-17T00:09:27.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 systemd[1]: Stopped Ceph osd.7 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:09:27.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 systemd[1]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.7.service: Consumed 5min 31.907s CPU time. 2024-09-17T00:09:27.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 systemd[1]: Starting Ceph osd.7 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:09:27.745 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 podman[62946]: 2024-09-17 00:09:27.38754019 +0000 UTC m=+0.016692202 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:09:27.745 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 podman[62946]: 2024-09-17 00:09:27.502921131 +0000 UTC m=+0.132073142 container create b1b9996037fa409fa11d0a89ca214845dfa8d9064541ccc329c93a8cf428a9de (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:09:27.745 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 podman[62946]: 2024-09-17 00:09:27.74744618 +0000 UTC m=+0.376598191 container init b1b9996037fa409fa11d0a89ca214845dfa8d9064541ccc329c93a8cf428a9de (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2) 2024-09-17T00:09:28.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 podman[62946]: 2024-09-17 00:09:27.751472836 +0000 UTC m=+0.380624852 container start b1b9996037fa409fa11d0a89ca214845dfa8d9064541ccc329c93a8cf428a9de (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-17T00:09:28.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 podman[62946]: 2024-09-17 00:09:27.864621739 +0000 UTC m=+0.493773751 container attach b1b9996037fa409fa11d0a89ca214845dfa8d9064541ccc329c93a8cf428a9de (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:09:28.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:09:28.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 bash[62946]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:09:28.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:09:28.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:27 smithi194 bash[62946]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-17T00:09:28.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 234445 8.45 MB/sec execute 552 sec latency 75.442 ms 2024-09-17T00:09:28.655 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T00:09:28.655 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T00:09:28.655 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T00:09:28.656 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T00:09:28.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-17T00:09:28.910 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate[62962]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T00:09:28.910 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 bash[62946]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-17T00:09:28.910 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:28 smithi194 podman[62946]: 2024-09-17 00:09:28.68845673 +0000 UTC m=+1.317608742 container died b1b9996037fa409fa11d0a89ca214845dfa8d9064541ccc329c93a8cf428a9de (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-17T00:09:29.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 234809 8.45 MB/sec execute 553 sec latency 68.443 ms 2024-09-17T00:09:29.608 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 podman[63092]: 2024-09-17 00:09:29.344227022 +0000 UTC m=+0.646374072 container remove b1b9996037fa409fa11d0a89ca214845dfa8d9064541ccc329c93a8cf428a9de (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=) 2024-09-17T00:09:29.609 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 podman[63148]: 2024-09-17 00:09:29.459454724 +0000 UTC m=+0.016913486 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:09:29.833 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:29 smithi007 ceph-mon[65487]: pgmap v275: 97 pgs: 5 active+undersized, 8 peering, 29 active+undersized+degraded, 55 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 437 op/s; 191/1575 objects degraded (12.127%) 2024-09-17T00:09:29.870 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:29 smithi194 ceph-mon[51517]: pgmap v275: 97 pgs: 5 active+undersized, 8 peering, 29 active+undersized+degraded, 55 active+clean; 324 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 3.8 MiB/s wr, 437 op/s; 191/1575 objects degraded (12.127%) 2024-09-17T00:09:29.870 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 podman[63148]: 2024-09-17 00:09:29.611744308 +0000 UTC m=+0.169203056 container create eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-17T00:09:30.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 podman[63148]: 2024-09-17 00:09:29.872989614 +0000 UTC m=+0.430448369 container init eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-17T00:09:30.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 podman[63148]: 2024-09-17 00:09:29.876348551 +0000 UTC m=+0.433807306 container start eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-17T00:09:30.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 ceph-osd[63167]: -- 172.21.15.194:0/3735859011 <== mon.1 v2:172.21.15.194:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a16d1381e0 con 0x55a16d112000 2024-09-17T00:09:30.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 bash[63148]: eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 2024-09-17T00:09:30.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:29 smithi194 systemd[1]: Started Ceph osd.7 for 5c15e83c-7486-11ef-bceb-c7b262605968. 2024-09-17T00:09:30.315 INFO:teuthology.orchestra.run.smithi007.stdout: 5 235133 8.45 MB/sec execute 554 sec latency 55.815 ms 2024-09-17T00:09:31.157 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:31.157 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:31.157 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:31 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:31.157 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:31 smithi007 ceph-mon[65487]: pgmap v276: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 376 op/s; 234/1611 objects degraded (14.525%) 2024-09-17T00:09:31.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 235406 8.44 MB/sec execute 555 sec latency 42.005 ms 2024-09-17T00:09:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:31 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:31.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:31 smithi194 ceph-mon[51517]: pgmap v276: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 326 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s wr, 376 op/s; 234/1611 objects degraded (14.525%) 2024-09-17T00:09:31.862 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:31 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:09:31.753+0000 7fa101f60640 -1 Falling back to public interface 2024-09-17T00:09:32.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 235678 8.43 MB/sec execute 556 sec latency 50.957 ms 2024-09-17T00:09:32.954 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:32 smithi007 ceph-mon[65487]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 511 op/s; 242/1659 objects degraded (14.587%) 2024-09-17T00:09:32.954 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:32 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 242/1659 objects degraded (14.587%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:33.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:32 smithi194 ceph-mon[51517]: pgmap v277: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 5.4 MiB/s wr, 511 op/s; 242/1659 objects degraded (14.587%) 2024-09-17T00:09:33.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:32 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 242/1659 objects degraded (14.587%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:33.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 235937 8.43 MB/sec execute 557 sec latency 112.932 ms 2024-09-17T00:09:34.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 236257 8.43 MB/sec execute 558 sec latency 105.320 ms 2024-09-17T00:09:35.042 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:34 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.042 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:34 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.042 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:34 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.042 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:34 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.042 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:34 smithi007 ceph-mon[65487]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 242/1659 objects degraded (14.587%) 2024-09-17T00:09:35.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:34 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:34 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:34 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:34 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:34 smithi194 ceph-mon[51517]: pgmap v278: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 333 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 4.6 MiB/s wr, 438 op/s; 242/1659 objects degraded (14.587%) 2024-09-17T00:09:35.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 236612 8.44 MB/sec execute 559 sec latency 87.060 ms 2024-09-17T00:09:36.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 236930 8.44 MB/sec execute 560 sec latency 45.325 ms 2024-09-17T00:09:37.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:36 smithi007 ceph-mon[65487]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.0 MiB/s wr, 470 op/s; 234/1581 objects degraded (14.801%) 2024-09-17T00:09:37.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:36 smithi194 ceph-mon[51517]: pgmap v279: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 327 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.2 KiB/s rd, 5.0 MiB/s wr, 470 op/s; 234/1581 objects degraded (14.801%) 2024-09-17T00:09:37.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 237194 8.44 MB/sec execute 561 sec latency 103.843 ms 2024-09-17T00:09:38.316 INFO:teuthology.orchestra.run.smithi007.stdout: 5 237419 8.43 MB/sec execute 562 sec latency 66.648 ms 2024-09-17T00:09:39.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 237759 8.44 MB/sec execute 563 sec latency 51.457 ms 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 319 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 435 op/s; 230/1569 objects degraded (14.659%); 4.5 KiB/s, 0 objects/s recovering 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:39.663 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all osd 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T00:09:39.664 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T00:09:39.665 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T00:09:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:09:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: pgmap v280: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 319 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.4 MiB/s wr, 435 op/s; 230/1569 objects degraded (14.659%); 4.5 KiB/s, 0 objects/s recovering 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all osd 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-17T00:09:39.910 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-17T00:09:39.911 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-09-17T00:09:40.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238003 8.43 MB/sec execute 564 sec latency 56.843 ms 2024-09-17T00:09:40.693 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T00:09:40.693 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 230/1569 objects degraded (14.659%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:40.694 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:40.694 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:40.694 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T00:09:40.694 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T00:09:40.694 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: osdmap e148: 8 total, 7 up, 8 in 2024-09-17T00:09:40.694 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi007.fkkfbl"]}]: dispatch 2024-09-17T00:09:40.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:09:40.788+0000 7fa101f60640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: Upgrade: Setting require_osd_release to 19 squid 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 230/1569 objects degraded (14.659%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: osdmap e148: 8 total, 7 up, 8 in 2024-09-17T00:09:40.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi007.fkkfbl"]}]: dispatch 2024-09-17T00:09:41.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238253 8.44 MB/sec execute 565 sec latency 154.476 ms 2024-09-17T00:09:41.737 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:41 smithi007 ceph-mon[65487]: Upgrade: It appears safe to stop mds.foofs.smithi007.fkkfbl 2024-09-17T00:09:41.738 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:41 smithi007 ceph-mon[65487]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 487 op/s; 230/1572 objects degraded (14.631%) 2024-09-17T00:09:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:41 smithi194 ceph-mon[51517]: Upgrade: It appears safe to stop mds.foofs.smithi007.fkkfbl 2024-09-17T00:09:41.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:41 smithi194 ceph-mon[51517]: pgmap v282: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 324 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.8 MiB/s wr, 487 op/s; 230/1572 objects degraded (14.631%) 2024-09-17T00:09:42.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238453 8.43 MB/sec execute 566 sec latency 92.725 ms 2024-09-17T00:09:42.658 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:09:42.325+0000 7fa101f60640 -1 osd.7 145 log_to_monitors true 2024-09-17T00:09:43.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: Upgrade: Updating mds.foofs.smithi007.fkkfbl 2024-09-17T00:09:43.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:43.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:09:43.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:43.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: Deploying daemon mds.foofs.smithi007.fkkfbl on smithi007 2024-09-17T00:09:43.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: from='osd.7 [v2:172.21.15.194:6824/2421401435,v1:172.21.15.194:6825/2421401435]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T00:09:43.058 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T00:09:43.058 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:42 smithi007 ceph-mon[65487]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 245/1626 objects degraded (15.068%) 2024-09-17T00:09:43.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238768 8.42 MB/sec execute 567 sec latency 80.625 ms 2024-09-17T00:09:43.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:09:43 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:09:43.045+0000 7fa0f9d0b640 -1 osd.7 145 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-17T00:09:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: Upgrade: Updating mds.foofs.smithi007.fkkfbl 2024-09-17T00:09:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi007.fkkfbl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:09:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:09:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: Deploying daemon mds.foofs.smithi007.fkkfbl on smithi007 2024-09-17T00:09:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: from='osd.7 [v2:172.21.15.194:6824/2421401435,v1:172.21.15.194:6825/2421401435]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T00:09:43.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-17T00:09:43.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:42 smithi194 ceph-mon[51517]: pgmap v283: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 328 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 406 op/s; 245/1626 objects degraded (15.068%) 2024-09-17T00:09:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:44 smithi007 ceph-mon[65487]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T00:09:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:44 smithi007 ceph-mon[65487]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T00:09:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:44 smithi007 ceph-mon[65487]: from='osd.7 [v2:172.21.15.194:6824/2421401435,v1:172.21.15.194:6825/2421401435]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:44 smithi007 ceph-mon[65487]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:44.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238968 8.41 MB/sec execute 568 sec latency 291.760 ms 2024-09-17T00:09:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:44 smithi194 ceph-mon[51517]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-17T00:09:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:44 smithi194 ceph-mon[51517]: osdmap e149: 8 total, 7 up, 8 in 2024-09-17T00:09:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:44 smithi194 ceph-mon[51517]: from='osd.7 [v2:172.21.15.194:6824/2421401435,v1:172.21.15.194:6825/2421401435]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:44.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:44 smithi194 ceph-mon[51517]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi194", "root=default"]}]: dispatch 2024-09-17T00:09:45.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:44 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[65483]: 2024-09-17T00:09:44.793+0000 7f23f4cde640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T00:09:45.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: osd.7 [v2:172.21.15.194:6824/2421401435,v1:172.21.15.194:6825/2421401435] boot 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: osdmap e150: 8 total, 8 up, 8 in 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 245/1626 objects degraded (15.068%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: osdmap e151: 8 total, 8 up, 8 in 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 369 op/s; 245/1626 objects degraded (15.068%) 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Standby daemon mds.foofs.smithi194.emredf assigned to filesystem foofs as rank 0 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T00:09:45.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T00:09:45.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T00:09:45.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:45 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:replay} 2024-09-17T00:09:45.317 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.40 MB/sec execute 569 sec latency 1290.563 ms 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: osd.7 [v2:172.21.15.194:6824/2421401435,v1:172.21.15.194:6825/2421401435] boot 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: osdmap e150: 8 total, 8 up, 8 in 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 245/1626 objects degraded (15.068%), 35 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: osdmap e151: 8 total, 8 up, 8 in 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: pgmap v287: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 328 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.6 MiB/s wr, 369 op/s; 245/1626 objects degraded (15.068%) 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T00:09:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: osdmap e152: 8 total, 8 up, 8 in 2024-09-17T00:09:45.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Standby daemon mds.foofs.smithi194.emredf assigned to filesystem foofs as rank 0 2024-09-17T00:09:45.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T00:09:45.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T00:09:45.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T00:09:45.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:45 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:replay} 2024-09-17T00:09:46.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.38 MB/sec execute 570 sec latency 2290.604 ms 2024-09-17T00:09:46.348 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:46 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:09:46.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:46 smithi007 ceph-mon[65487]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T00:09:46.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:46 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:09:46.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:46 smithi194 ceph-mon[51517]: osdmap e153: 8 total, 8 up, 8 in 2024-09-17T00:09:47.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.37 MB/sec execute 571 sec latency 3290.637 ms 2024-09-17T00:09:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:47 smithi007 ceph-mon[65487]: pgmap v290: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 peering, 2 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized, 13 active+undersized+degraded, 72 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.4 MiB/s wr, 356 op/s; 160/1665 objects degraded (9.610%); 2.4 MiB/s, 23 objects/s recovering 2024-09-17T00:09:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:47 smithi007 ceph-mon[65487]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T00:09:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:47 smithi007 ceph-mon[65487]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T00:09:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:47 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.7:6834/2256561135,v1:172.21.15.7:6835/2256561135] up:boot 2024-09-17T00:09:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:47 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:replay} 1 up:standby 2024-09-17T00:09:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:47 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:09:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:47 smithi194 ceph-mon[51517]: pgmap v290: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 3 peering, 2 active+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized, 13 active+undersized+degraded, 72 active+clean; 331 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 2.4 MiB/s wr, 356 op/s; 160/1665 objects degraded (9.610%); 2.4 MiB/s, 23 objects/s recovering 2024-09-17T00:09:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:47 smithi194 ceph-mon[51517]: osdmap e154: 8 total, 8 up, 8 in 2024-09-17T00:09:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:47 smithi194 ceph-mon[51517]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T00:09:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:47 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.7:6834/2256561135,v1:172.21.15.7:6835/2256561135] up:boot 2024-09-17T00:09:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:47 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:replay} 1 up:standby 2024-09-17T00:09:48.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:47 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:09:48.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.36 MB/sec execute 572 sec latency 4290.673 ms 2024-09-17T00:09:49.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:48 smithi007 ceph-mon[65487]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T00:09:49.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:48 smithi194 ceph-mon[51517]: osdmap e155: 8 total, 8 up, 8 in 2024-09-17T00:09:49.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.34 MB/sec execute 573 sec latency 5290.707 ms 2024-09-17T00:09:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:49 smithi007 ceph-mon[65487]: pgmap v293: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.7 MiB/s wr, 367 op/s; 27/1659 objects degraded (1.627%); 4.9 MiB/s, 58 objects/s recovering 2024-09-17T00:09:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:49 smithi007 ceph-mon[65487]: Health check update: Degraded data redundancy: 27/1659 objects degraded (1.627%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:49 smithi194 ceph-mon[51517]: pgmap v293: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 1 active+undersized+remapped, 1 active+recovery_wait+degraded, 91 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 2.7 MiB/s wr, 367 op/s; 27/1659 objects degraded (1.627%); 4.9 MiB/s, 58 objects/s recovering 2024-09-17T00:09:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:49 smithi194 ceph-mon[51517]: Health check update: Degraded data redundancy: 27/1659 objects degraded (1.627%), 2 pgs degraded (PG_DEGRADED) 2024-09-17T00:09:50.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.33 MB/sec execute 574 sec latency 6290.741 ms 2024-09-17T00:09:51.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.31 MB/sec execute 575 sec latency 7290.775 ms 2024-09-17T00:09:51.491 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:09:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:51 smithi007 ceph-mon[65487]: pgmap v294: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 93 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 1.8 MiB/s wr, 249 op/s; 22/1659 objects degraded (1.326%); 3.3 MiB/s, 40 objects/s recovering 2024-09-17T00:09:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:51 smithi007 ceph-mon[65487]: reconnect by client.24367 172.21.15.194:0/2491978951 after 0 2024-09-17T00:09:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:51 smithi007 ceph-mon[65487]: reconnect by client.14530 172.21.15.7:0/2969757684 after 0 2024-09-17T00:09:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:51 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:reconnect 2024-09-17T00:09:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:51 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:reconnect} 1 up:standby 2024-09-17T00:09:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:51 smithi194 ceph-mon[51517]: pgmap v294: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 3 peering, 93 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 MiB/s rd, 1.8 MiB/s wr, 249 op/s; 22/1659 objects degraded (1.326%); 3.3 MiB/s, 40 objects/s recovering 2024-09-17T00:09:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:51 smithi194 ceph-mon[51517]: reconnect by client.24367 172.21.15.194:0/2491978951 after 0 2024-09-17T00:09:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:51 smithi194 ceph-mon[51517]: reconnect by client.14530 172.21.15.7:0/2969757684 after 0 2024-09-17T00:09:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:51 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:reconnect 2024-09-17T00:09:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:51 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:reconnect} 1 up:standby 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (9m) 2m ago 18m 26.3M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (5m) 2m ago 18m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (5m) 18s ago 17m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (9m) 2m ago 17m 53.4M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (12m) 2m ago 12m 37.5M - 17.2.0 e1d6a67b021e cbe778142fb6 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (12m) 18s ago 12m 15.5M - 17.2.0 e1d6a67b021e ff18d270e3f5 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (8m) 2m ago 19m 503M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (7m) 18s ago 17m 442M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (7m) 2m ago 19m 50.1M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (6m) 18s ago 17m 42.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:09:52.178 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (11m) 2m ago 11m 270M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (11m) 18s ago 11m 88.6M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (8m) 2m ago 17m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (8m) 18s ago 17m 16.2M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (5m) 2m ago 16m 321M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (4m) 2m ago 16m 258M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (3m) 2m ago 15m 215M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (2m) 2m ago 15m 11.1M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (2m) 18s ago 14m 306M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (93s) 18s ago 14m 270M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (57s) 18s ago 14m 234M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 81dd179cd962 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (22s) 18s ago 13m 12.0M 2704M 19.1.1-330-gc2ddf972 ebb0615149de eac97979e96c 2024-09-17T00:09:52.179 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (7m) 2m ago 17m 66.3M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:09:52.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.30 MB/sec execute 576 sec latency 8290.814 ms 2024-09-17T00:09:52.597 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:09:52.597 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:09:52.597 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:09:52.597 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:52.597 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:09:52.597 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 12 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:09:52.598 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:09:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:52 smithi007 ceph-mon[65487]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:52 smithi007 ceph-mon[65487]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:52 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:rejoin 2024-09-17T00:09:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:52 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:rejoin} 1 up:standby 2024-09-17T00:09:52.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:52 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/1170627819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:52.972 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:09:52.972 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:09:52.972 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:09:52.972 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:09:52.972 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:09:52.972 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr", 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: "osd" 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "14/23 daemons upgraded", 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:09:52.973 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:09:53.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:52 smithi194 ceph-mon[51517]: from='client.34444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:53.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:52 smithi194 ceph-mon[51517]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:53.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:52 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:rejoin 2024-09-17T00:09:53.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:52 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:rejoin} 1 up:standby 2024-09-17T00:09:53.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:52 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/1170627819' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:09:53.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 238969 8.28 MB/sec execute 577 sec latency 9290.869 ms 2024-09-17T00:09:53.413 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_WARN 1 filesystem is degraded 2024-09-17T00:09:53.414 INFO:teuthology.orchestra.run.smithi007.stdout:[WRN] FS_DEGRADED: 1 filesystem is degraded 2024-09-17T00:09:53.414 INFO:teuthology.orchestra.run.smithi007.stdout: fs foofs is degraded 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: pgmap v295: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 612 KiB/s wr, 45 op/s; 2.4 MiB/s, 35 objects/s recovering 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1659 objects degraded (1.326%), 1 pg degraded) 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:clientreplay 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:clientreplay} 1 up:standby 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: daemon mds.foofs.smithi194.emredf is now active in filesystem foofs as rank 0 2024-09-17T00:09:54.088 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:53 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3353955034' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:09:54.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: pgmap v295: 97 pgs: 97 active+clean; 330 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 612 KiB/s wr, 45 op/s; 2.4 MiB/s, 35 objects/s recovering 2024-09-17T00:09:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/1659 objects degraded (1.326%), 1 pg degraded) 2024-09-17T00:09:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:clientreplay 2024-09-17T00:09:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi194.emredf=up:clientreplay} 1 up:standby 2024-09-17T00:09:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: daemon mds.foofs.smithi194.emredf is now active in filesystem foofs as rank 0 2024-09-17T00:09:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:53 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3353955034' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:09:54.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239025 8.27 MB/sec execute 578 sec latency 9863.058 ms 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:active 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi194.emredf=up:active} 1 up:standby 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:54.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:54.850 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:54 smithi007 ceph-mon[65487]: pgmap v296: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 219 KiB/s wr, 24 op/s; 1.4 MiB/s, 21 objects/s recovering 2024-09-17T00:09:55.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: from='client.34460 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.194:6832/2212315854,v1:172.21.15.194:6833/2212315854] up:active 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: fsmap foofs:1 {0=foofs.smithi194.emredf=up:active} 1 up:standby 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:09:55.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:54 smithi194 ceph-mon[51517]: pgmap v296: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 219 KiB/s wr, 24 op/s; 1.4 MiB/s, 21 objects/s recovering 2024-09-17T00:09:55.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239143 8.26 MB/sec execute 579 sec latency 126.487 ms 2024-09-17T00:09:56.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239258 8.25 MB/sec execute 580 sec latency 117.525 ms 2024-09-17T00:09:56.942 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:56 smithi007 ceph-mon[65487]: pgmap v297: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 418 KiB/s wr, 58 op/s; 1.2 MiB/s, 17 objects/s recovering 2024-09-17T00:09:57.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:56 smithi194 ceph-mon[51517]: pgmap v297: 97 pgs: 97 active+clean; 329 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 418 KiB/s wr, 58 op/s; 1.2 MiB/s, 17 objects/s recovering 2024-09-17T00:09:57.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239369 8.25 MB/sec execute 581 sec latency 134.428 ms 2024-09-17T00:09:58.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:57 smithi007 ceph-mon[65487]: daemon mds.foofs.smithi007.fkkfbl restarted 2024-09-17T00:09:58.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:57 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:58.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:57 smithi007 ceph-mon[65487]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T00:09:58.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:57 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:58.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:57 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:57 smithi194 ceph-mon[51517]: daemon mds.foofs.smithi007.fkkfbl restarted 2024-09-17T00:09:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:57 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:57 smithi194 ceph-mon[51517]: osdmap e156: 8 total, 8 up, 8 in 2024-09-17T00:09:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:57 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:09:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:57 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:09:58.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239508 8.24 MB/sec execute 582 sec latency 168.662 ms 2024-09-17T00:09:59.046 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:58 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:boot 2024-09-17T00:09:59.046 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:58 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi194.emredf=up:active} 1 up:standby 2024-09-17T00:09:59.046 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:58 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:09:59.046 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:09:58 smithi007 ceph-mon[65487]: pgmap v299: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 445 KiB/s wr, 60 op/s; 101 KiB/s, 2 objects/s recovering 2024-09-17T00:09:59.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239653 8.23 MB/sec execute 583 sec latency 146.773 ms 2024-09-17T00:09:59.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:58 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:boot 2024-09-17T00:09:59.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:58 smithi194 ceph-mon[51517]: fsmap foofs:1 {0=foofs.smithi194.emredf=up:active} 1 up:standby 2024-09-17T00:09:59.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:58 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:09:59.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:09:58 smithi194 ceph-mon[51517]: pgmap v299: 97 pgs: 97 active+clean; 323 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 445 KiB/s wr, 60 op/s; 101 KiB/s, 2 objects/s recovering 2024-09-17T00:10:00.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239764 8.22 MB/sec execute 584 sec latency 126.931 ms 2024-09-17T00:10:00.860 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:00 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:00.860 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:00 smithi007 ceph-mon[65487]: overall HEALTH_OK 2024-09-17T00:10:01.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:00 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:01.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:00 smithi194 ceph-mon[51517]: overall HEALTH_OK 2024-09-17T00:10:01.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 239891 8.21 MB/sec execute 585 sec latency 101.513 ms 2024-09-17T00:10:01.898 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:01 smithi007 ceph-mon[65487]: pgmap v300: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 921 KiB/s wr, 135 op/s; 92 KiB/s, 2 objects/s recovering 2024-09-17T00:10:02.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:01 smithi194 ceph-mon[51517]: pgmap v300: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 921 KiB/s wr, 135 op/s; 92 KiB/s, 2 objects/s recovering 2024-09-17T00:10:02.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240048 8.21 MB/sec execute 586 sec latency 105.827 ms 2024-09-17T00:10:03.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240190 8.20 MB/sec execute 587 sec latency 114.709 ms 2024-09-17T00:10:03.991 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:03 smithi007 ceph-mon[65487]: pgmap v301: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.4 MiB/s wr, 159 op/s 2024-09-17T00:10:04.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:03 smithi194 ceph-mon[51517]: pgmap v301: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.4 MiB/s wr, 159 op/s 2024-09-17T00:10:04.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240292 8.20 MB/sec execute 588 sec latency 135.039 ms 2024-09-17T00:10:05.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240411 8.19 MB/sec execute 589 sec latency 180.462 ms 2024-09-17T00:10:05.660 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:05 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.660 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:05 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.660 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:05 smithi007 ceph-mon[65487]: pgmap v302: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 1.6 MiB/s wr, 164 op/s 2024-09-17T00:10:05.661 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:05 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.661 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:05 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:05 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:05 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:05 smithi194 ceph-mon[51517]: pgmap v302: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 1.6 MiB/s wr, 164 op/s 2024-09-17T00:10:05.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:05 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:05.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:05 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:06.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240606 8.18 MB/sec execute 590 sec latency 95.716 ms 2024-09-17T00:10:07.116 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:06 smithi007 ceph-mon[65487]: pgmap v303: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 1.8 MiB/s wr, 205 op/s 2024-09-17T00:10:07.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:06 smithi194 ceph-mon[51517]: pgmap v303: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 1.8 MiB/s wr, 205 op/s 2024-09-17T00:10:07.318 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240730 8.17 MB/sec execute 591 sec latency 149.425 ms 2024-09-17T00:10:08.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240843 8.16 MB/sec execute 592 sec latency 151.140 ms 2024-09-17T00:10:08.979 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:08 smithi007 ceph-mon[65487]: pgmap v304: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.9 MiB/s wr, 191 op/s 2024-09-17T00:10:09.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:08 smithi194 ceph-mon[51517]: pgmap v304: 97 pgs: 97 active+clean; 322 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.9 MiB/s wr, 191 op/s 2024-09-17T00:10:09.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 240919 8.15 MB/sec execute 593 sec latency 126.650 ms 2024-09-17T00:10:10.047 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:10.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:10.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 241009 8.14 MB/sec execute 594 sec latency 152.750 ms 2024-09-17T00:10:11.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 241144 8.14 MB/sec execute 595 sec latency 115.249 ms 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: Detected new or changed devices on smithi007 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:11.331 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi194.emredf"]}]: dispatch 2024-09-17T00:10:11.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: Upgrade: It appears safe to stop mds.foofs.smithi194.emredf 2024-09-17T00:10:11.332 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:11 smithi007 ceph-mon[65487]: pgmap v305: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.1 MiB/s wr, 227 op/s 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: Detected new or changed devices on smithi007 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:11.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds ok-to-stop", "ids": ["foofs.smithi194.emredf"]}]: dispatch 2024-09-17T00:10:11.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: Upgrade: It appears safe to stop mds.foofs.smithi194.emredf 2024-09-17T00:10:11.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:11 smithi194 ceph-mon[51517]: pgmap v305: 97 pgs: 97 active+clean; 325 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.1 MiB/s wr, 227 op/s 2024-09-17T00:10:12.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 241288 8.13 MB/sec execute 596 sec latency 161.248 ms 2024-09-17T00:10:13.153 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-mon[65487]: Upgrade: Updating mds.foofs.smithi194.emredf 2024-09-17T00:10:13.153 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:13.153 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:10:13.153 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:13.153 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-mon[65487]: Deploying daemon mds.foofs.smithi194.emredf on smithi194 2024-09-17T00:10:13.153 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-mon[65487]: pgmap v306: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 1.9 MiB/s wr, 175 op/s 2024-09-17T00:10:13.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 241375 8.12 MB/sec execute 597 sec latency 114.559 ms 2024-09-17T00:10:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:13 smithi194 ceph-mon[51517]: Upgrade: Updating mds.foofs.smithi194.emredf 2024-09-17T00:10:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi194.emredf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-17T00:10:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:13 smithi194 ceph-mon[51517]: Deploying daemon mds.foofs.smithi194.emredf on smithi194 2024-09-17T00:10:13.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:13 smithi194 ceph-mon[51517]: pgmap v306: 97 pgs: 97 active+clean; 326 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 1.9 MiB/s wr, 175 op/s 2024-09-17T00:10:14.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 241431 8.11 MB/sec execute 598 sec latency 337.375 ms 2024-09-17T00:10:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:13 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[65483]: 2024-09-17T00:10:13.994+0000 7f23f4cde640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T00:10:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T00:10:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T00:10:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T00:10:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: Standby daemon mds.foofs.smithi007.fkkfbl assigned to filesystem foofs as rank 0 2024-09-17T00:10:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T00:10:14.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T00:10:14.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T00:10:14.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:14 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:replay} 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: osdmap e157: 8 total, 8 up, 8 in 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: Standby daemon mds.foofs.smithi007.fkkfbl assigned to filesystem foofs as rank 0 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-17T00:10:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-17T00:10:14.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:14 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:replay} 2024-09-17T00:10:15.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 241431 8.10 MB/sec execute 599 sec latency 1337.424 ms 2024-09-17T00:10:15.348 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:15 smithi007 ceph-mon[65487]: pgmap v308: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 1.8 MiB/s wr, 183 op/s 2024-09-17T00:10:15.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:15 smithi194 ceph-mon[51517]: pgmap v308: 97 pgs: 97 active+clean; 327 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 1.8 MiB/s wr, 183 op/s 2024-09-17T00:10:16.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 cleanup 600 sec 2024-09-17T00:10:16.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:16 smithi007 ceph-mon[65487]: reconnect by client.24367 172.21.15.194:0/2491978951 after 0 2024-09-17T00:10:16.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:16 smithi007 ceph-mon[65487]: reconnect by client.14530 172.21.15.7:0/2969757684 after 0 2024-09-17T00:10:16.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:16 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:reconnect 2024-09-17T00:10:16.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:16 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:reconnect} 2024-09-17T00:10:16.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:16 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:10:16.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:16 smithi194 ceph-mon[51517]: reconnect by client.24367 172.21.15.194:0/2491978951 after 0 2024-09-17T00:10:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:16 smithi194 ceph-mon[51517]: reconnect by client.14530 172.21.15.7:0/2969757684 after 0 2024-09-17T00:10:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:16 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:reconnect 2024-09-17T00:10:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:16 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:reconnect} 2024-09-17T00:10:16.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:16 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:10:17.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 cleanup 601 sec 2024-09-17T00:10:17.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:17 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:rejoin 2024-09-17T00:10:17.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:17 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:rejoin} 2024-09-17T00:10:17.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:17 smithi007 ceph-mon[65487]: pgmap v309: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 132 op/s 2024-09-17T00:10:17.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:17 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:rejoin 2024-09-17T00:10:17.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:17 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:rejoin} 2024-09-17T00:10:17.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:17 smithi194 ceph-mon[51517]: pgmap v309: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.6 MiB/s wr, 132 op/s 2024-09-17T00:10:18.319 INFO:teuthology.orchestra.run.smithi007.stdout: 5 cleanup 602 sec 2024-09-17T00:10:18.325 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:18 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:clientreplay 2024-09-17T00:10:18.325 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:18 smithi007 ceph-mon[65487]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:clientreplay} 2024-09-17T00:10:18.325 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:18 smithi007 ceph-mon[65487]: daemon mds.foofs.smithi007.fkkfbl is now active in filesystem foofs as rank 0 2024-09-17T00:10:18.404 INFO:teuthology.orchestra.run.smithi007.stdout: 0 cleanup 602 sec 2024-09-17T00:10:18.404 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-17T00:10:18.404 INFO:teuthology.orchestra.run.smithi007.stdout: Operation Count AvgLat MaxLat 2024-09-17T00:10:18.404 INFO:teuthology.orchestra.run.smithi007.stdout: ---------------------------------------- 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: NTCreateX 155320 5.476 9863.052 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Close 114100 0.142 21.328 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Rename 6584 12.455 4142.992 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Unlink 31335 8.037 1060.675 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Qpathinfo 140913 1.403 4121.998 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Qfileinfo 24606 0.007 0.825 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Qfsinfo 25800 1.221 2051.654 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Sfileinfo 12629 15.201 2311.908 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: Find 54437 1.009 93.581 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: WriteX 76762 17.275 9830.786 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: ReadX 244116 0.010 4.443 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: LockX 510 0.582 17.667 2024-09-17T00:10:18.405 INFO:teuthology.orchestra.run.smithi007.stdout: UnlockX 510 0.449 21.392 2024-09-17T00:10:18.406 INFO:teuthology.orchestra.run.smithi007.stdout: Flush 10862 0.003 0.038 2024-09-17T00:10:18.406 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-17T00:10:18.406 INFO:teuthology.orchestra.run.smithi007.stdout:Throughput 8.09575 MB/sec 5 clients 5 procs max_latency=9863.058 ms 2024-09-17T00:10:18.407 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-17T00:10:18.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:18 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:clientreplay 2024-09-17T00:10:18.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:18 smithi194 ceph-mon[51517]: fsmap foofs:1/1 {0=foofs.smithi007.fkkfbl=up:clientreplay} 2024-09-17T00:10:18.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:18 smithi194 ceph-mon[51517]: daemon mds.foofs.smithi007.fkkfbl is now active in filesystem foofs as rank 0 2024-09-17T00:10:18.446 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-17T00:10:18.572 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-17T00:10:18.597 INFO:teuthology.orchestra.run.smithi007.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-17T00:10:18.598 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-17T00:10:18.598 INFO:teuthology.orchestra.run.smithi007.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-17T00:10:18.598 INFO:teuthology.orchestra.run.smithi007.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-17T00:10:18.699 INFO:teuthology.orchestra.run.smithi007.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-17T00:10:18.699 INFO:teuthology.orchestra.run.smithi007.stdout:releasing clients 2024-09-17T00:10:19.124 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:19 smithi007 ceph-mon[65487]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T00:10:19.124 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:19 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:active 2024-09-17T00:10:19.124 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:19 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 2024-09-17T00:10:19.124 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:19 smithi007 ceph-mon[65487]: pgmap v310: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.4 MiB/s wr, 150 op/s 2024-09-17T00:10:19.236 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:19 smithi194 ceph-mon[51517]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-17T00:10:19.236 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:19 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938] up:active 2024-09-17T00:10:19.236 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:19 smithi194 ceph-mon[51517]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 2024-09-17T00:10:19.236 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:19 smithi194 ceph-mon[51517]: pgmap v310: 97 pgs: 97 active+clean; 321 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 1.4 MiB/s wr, 150 op/s 2024-09-17T00:10:20.699 INFO:teuthology.orchestra.run.smithi007.stdout: 5 284 22.37 MB/sec execute 1 sec latency 423.231 ms 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: Cluster is now healthy 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: mds.? [v2:172.21.15.194:6832/2003743806,v1:172.21.15.194:6833/2003743806] up:boot 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:20.728 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:20.729 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:20 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: Cluster is now healthy 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: mds.? [v2:172.21.15.194:6832/2003743806,v1:172.21.15.194:6833/2003743806] up:boot 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: fsmap foofs:1 {0=foofs.smithi007.fkkfbl=up:active} 1 up:standby 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:20.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:20 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:21.699 INFO:teuthology.orchestra.run.smithi007.stdout: 5 417 21.59 MB/sec execute 2 sec latency 106.273 ms 2024-09-17T00:10:21.844 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:21 smithi007 ceph-mon[65487]: pgmap v311: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 105 op/s 2024-09-17T00:10:22.051 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:21 smithi194 ceph-mon[51517]: pgmap v311: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 2.7 MiB/s wr, 105 op/s 2024-09-17T00:10:22.699 INFO:teuthology.orchestra.run.smithi007.stdout: 5 579 21.80 MB/sec execute 3 sec latency 395.108 ms 2024-09-17T00:10:23.699 INFO:teuthology.orchestra.run.smithi007.stdout: 5 803 19.21 MB/sec execute 4 sec latency 232.748 ms 2024-09-17T00:10:23.915 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:10:24.010 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:23 smithi007 ceph-mon[65487]: pgmap v312: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.9 MiB/s wr, 205 op/s 2024-09-17T00:10:24.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:23 smithi194 ceph-mon[51517]: pgmap v312: 97 pgs: 97 active+clean; 328 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 5.9 MiB/s wr, 205 op/s 2024-09-17T00:10:24.699 INFO:teuthology.orchestra.run.smithi007.stdout: 5 cleanup 5 sec 2024-09-17T00:10:24.988 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:10:24.988 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (9m) 20s ago 18m 26.4M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:10:24.988 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (6m) 20s ago 18m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:10:24.988 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (6m) 1s ago 18m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:10:24.988 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (9m) 20s ago 18m 53.2M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:10:24.988 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (28s) 20s ago 12m 12.0M - 19.1.1-330-gc2ddf972 ebb0615149de 2fd65c3902d3 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (4s) 1s ago 12m 12.8M - 19.1.1-330-gc2ddf972 ebb0615149de 0868978d8471 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (8m) 20s ago 19m 508M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (7m) 1s ago 18m 443M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (7m) 20s ago 20m 58.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (7m) 1s ago 17m 44.7M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (12m) 20s ago 12m 278M - 4.0 e1d6a67b021e 8b191655c96d 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (12m) 1s ago 12m 88.7M - 4.0 e1d6a67b021e 406a3f3a0797 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (9m) 20s ago 18m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (9m) 1s ago 17m 16.6M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (5m) 20s ago 17m 359M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (5m) 20s ago 16m 298M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:10:24.989 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (4m) 20s ago 16m 345M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:10:24.990 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (3m) 20s ago 15m 333M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:10:24.990 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (2m) 1s ago 15m 329M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:10:24.990 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (2m) 1s ago 14m 297M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:10:24.990 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (89s) 1s ago 14m 258M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 81dd179cd962 2024-09-17T00:10:24.990 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (55s) 1s ago 14m 310M 2704M 19.1.1-330-gc2ddf972 ebb0615149de eac97979e96c 2024-09-17T00:10:24.990 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (8m) 20s ago 17m 63.9M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: 0 cleanup 5 sec 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: Operation Count AvgLat MaxLat 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: ---------------------------------------- 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: NTCreateX 658 9.330 106.453 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: Close 612 13.053 395.103 2024-09-17T00:10:25.017 INFO:teuthology.orchestra.run.smithi007.stdout: Rename 27 7.076 47.413 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Unlink 52 10.877 87.859 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Qpathinfo 233 1.256 25.788 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Qfileinfo 357 18.635 312.791 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Qfsinfo 116 0.551 3.319 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Sfileinfo 101 14.953 23.086 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Find 142 0.877 2.398 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: WriteX 1753 0.027 0.070 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: ReadX 507 0.023 1.236 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: Flush 114 15.499 64.292 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-17T00:10:25.018 INFO:teuthology.orchestra.run.smithi007.stdout:Throughput 19.2065 MB/sec 5 clients 5 procs max_latency=423.231 ms 2024-09-17T00:10:25.020 DEBUG:teuthology.parallel:result is None 2024-09-17T00:10:25.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:25.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:25.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:25.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:24 smithi007 ceph-mon[65487]: pgmap v313: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.0 MiB/s wr, 204 op/s 2024-09-17T00:10:25.229 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='client.34476 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:25.229 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.229 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.229 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='client.34480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:25.229 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.230 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:25.230 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:25.230 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:24 smithi194 ceph-mon[51517]: pgmap v313: 97 pgs: 97 active+clean; 323 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 7.0 MiB/s wr, 204 op/s 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:25.420 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:10:25.421 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:10:25.421 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:25.421 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:10:25.421 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T00:10:25.421 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:10:25.421 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:10:25.787 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "mds", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "osd", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "16/23 daemons upgraded", 2024-09-17T00:10:25.788 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading mds daemons", 2024-09-17T00:10:25.789 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:10:25.789 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:10:26.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:25 smithi007 ceph-mon[65487]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:26.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:25 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3664565757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:26.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:25 smithi007 ceph-mon[65487]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:26.242 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:10:26.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:25 smithi194 ceph-mon[51517]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:26.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:25 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3664565757' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:26.306 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:25 smithi194 ceph-mon[51517]: from='client.34492 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:27.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:26 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/2445508755' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:10:27.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:26 smithi194 ceph-mon[51517]: pgmap v314: 97 pgs: 97 active+clean; 311 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 9.3 MiB/s wr, 231 op/s 2024-09-17T00:10:27.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:26 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/2445508755' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:10:27.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:26 smithi007 ceph-mon[65487]: pgmap v314: 97 pgs: 97 active+clean; 311 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 9.3 MiB/s wr, 231 op/s 2024-09-17T00:10:29.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: Detected new or changed devices on smithi194 2024-09-17T00:10:29.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.350 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all mds 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi007.fkkfbl"}]': finished 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.emredf"}]': finished 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all rgw 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T00:10:29.351 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all iscsi 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.352 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:29 smithi007 ceph-mon[65487]: pgmap v315: 97 pgs: 97 active+clean; 296 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 9.9 MiB/s wr, 284 op/s 2024-09-17T00:10:29.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: Detected new or changed devices on smithi194 2024-09-17T00:10:29.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:29.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:10:29.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all mds 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi007.fkkfbl"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi007.fkkfbl"}]': finished 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.emredf"}]: dispatch 2024-09-17T00:10:29.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi194.emredf"}]': finished 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all rgw 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all rbd-mirror 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all ceph-exporter 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all iscsi 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:29.411 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:29 smithi194 ceph-mon[51517]: pgmap v315: 97 pgs: 97 active+clean; 296 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 9.9 MiB/s wr, 284 op/s 2024-09-17T00:10:30.992 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Upgrade: Updating nfs.foo.0.0.smithi007.lardmp (1/2) 2024-09-17T00:10:30.992 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:30.992 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Creating key for client.nfs.foo.0.0.smithi007.lardmp 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Rados config object exists: conf-nfs.foo 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Creating key for client.nfs.foo.0.0.smithi007.lardmp-rgw 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Bind address in nfs.foo.0.0.smithi007.lardmp's ganesha conf is defaulting to empty 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:30.993 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:30 smithi007 ceph-mon[65487]: Deploying daemon nfs.foo.0.0.smithi007.lardmp on smithi007 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Upgrade: Updating nfs.foo.0.0.smithi007.lardmp (1/2) 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Creating key for client.nfs.foo.0.0.smithi007.lardmp 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:31.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Rados config object exists: conf-nfs.foo 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Creating key for client.nfs.foo.0.0.smithi007.lardmp-rgw 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi007.lardmp-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Bind address in nfs.foo.0.0.smithi007.lardmp's ganesha conf is defaulting to empty 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:31.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:30 smithi194 ceph-mon[51517]: Deploying daemon nfs.foo.0.0.smithi007.lardmp on smithi007 2024-09-17T00:10:31.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:31 smithi007 ceph-mon[65487]: pgmap v316: 97 pgs: 97 active+clean; 260 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 9.7 MiB/s wr, 267 op/s 2024-09-17T00:10:32.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:31 smithi194 ceph-mon[51517]: pgmap v316: 97 pgs: 97 active+clean; 260 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 9.7 MiB/s wr, 267 op/s 2024-09-17T00:10:33.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:32 smithi007 ceph-mon[65487]: pgmap v317: 97 pgs: 97 active+clean; 217 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 8.3 MiB/s wr, 262 op/s 2024-09-17T00:10:33.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:32 smithi194 ceph-mon[51517]: pgmap v317: 97 pgs: 97 active+clean; 217 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 8.3 MiB/s wr, 262 op/s 2024-09-17T00:10:34.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:33 smithi007 ceph-mon[65487]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.7:0/2969757684"}]: dispatch 2024-09-17T00:10:34.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:33 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:34.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:33 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:33 smithi194 ceph-mon[51517]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.7:0/2969757684"}]: dispatch 2024-09-17T00:10:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:33 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:34.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:33 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:34 smithi194 ceph-mon[51517]: Evicting (and blocklisting) client session 14530 (172.21.15.7:0/2969757684) 2024-09-17T00:10:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:34 smithi194 ceph-mon[51517]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.7:0/2969757684"}]': finished 2024-09-17T00:10:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:34 smithi194 ceph-mon[51517]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T00:10:35.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:34 smithi194 ceph-mon[51517]: pgmap v319: 97 pgs: 97 active+clean; 215 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 179 op/s 2024-09-17T00:10:35.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:34 smithi007 ceph-mon[65487]: Evicting (and blocklisting) client session 14530 (172.21.15.7:0/2969757684) 2024-09-17T00:10:35.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:34 smithi007 ceph-mon[65487]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.7:0/2969757684"}]': finished 2024-09-17T00:10:35.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:34 smithi007 ceph-mon[65487]: osdmap e158: 8 total, 8 up, 8 in 2024-09-17T00:10:35.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:34 smithi007 ceph-mon[65487]: pgmap v319: 97 pgs: 97 active+clean; 215 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.7 MiB/s wr, 179 op/s 2024-09-17T00:10:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Upgrade: Updating nfs.foo.1.0.smithi194.wpbzoi (2/2) 2024-09-17T00:10:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi 2024-09-17T00:10:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T00:10:36.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Rados config object exists: conf-nfs.foo 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi-rgw 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Bind address in nfs.foo.1.0.smithi194.wpbzoi's ganesha conf is defaulting to empty 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:36.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:36 smithi194 ceph-mon[51517]: Deploying daemon nfs.foo.1.0.smithi194.wpbzoi on smithi194 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Upgrade: Updating nfs.foo.1.0.smithi194.wpbzoi (2/2) 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:36.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Rados config object exists: conf-nfs.foo 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Creating key for client.nfs.foo.1.0.smithi194.wpbzoi-rgw 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi194.wpbzoi-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Bind address in nfs.foo.1.0.smithi194.wpbzoi's ganesha conf is defaulting to empty 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:36.600 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:36 smithi007 ceph-mon[65487]: Deploying daemon nfs.foo.1.0.smithi194.wpbzoi on smithi194 2024-09-17T00:10:37.334 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:37 smithi194 ceph-mon[51517]: pgmap v320: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.1 MiB/s wr, 128 op/s 2024-09-17T00:10:37.599 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:37 smithi007 ceph-mon[65487]: pgmap v320: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.1 MiB/s wr, 128 op/s 2024-09-17T00:10:39.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:38 smithi194 ceph-mon[51517]: pgmap v321: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 36 KiB/s wr, 34 op/s 2024-09-17T00:10:39.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:38 smithi007 ceph-mon[65487]: pgmap v321: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 36 KiB/s wr, 34 op/s 2024-09-17T00:10:40.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:40.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:40.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:40.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:40 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:40 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:41.344 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:41 smithi194 ceph-mon[51517]: pgmap v322: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 5.9 KiB/s wr, 21 op/s 2024-09-17T00:10:41.348 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:41 smithi007 ceph-mon[65487]: pgmap v322: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 KiB/s rd, 5.9 KiB/s wr, 21 op/s 2024-09-17T00:10:43.091 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.092 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.092 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:42 smithi194 ceph-mon[51517]: Evicting (and blocklisting) client session 24367 (172.21.15.194:0/2491978951) 2024-09-17T00:10:43.092 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:42 smithi194 ceph-mon[51517]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491978951"}]: dispatch 2024-09-17T00:10:43.092 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.092 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:42 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.093 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.093 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.093 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:42 smithi007 ceph-mon[65487]: Evicting (and blocklisting) client session 24367 (172.21.15.194:0/2491978951) 2024-09-17T00:10:43.094 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:42 smithi007 ceph-mon[65487]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd=[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491978951"}]: dispatch 2024-09-17T00:10:43.094 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:43.094 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:42 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:44.018 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:43 smithi194 ceph-mon[51517]: pgmap v323: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 6.8 KiB/s wr, 9 op/s 2024-09-17T00:10:44.018 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:43 smithi194 ceph-mon[51517]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491978951"}]': finished 2024-09-17T00:10:44.018 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:43 smithi194 ceph-mon[51517]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T00:10:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:43 smithi007 ceph-mon[65487]: pgmap v323: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 6.8 KiB/s wr, 9 op/s 2024-09-17T00:10:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:43 smithi007 ceph-mon[65487]: from='mds.? [v2:172.21.15.7:6834/1021580938,v1:172.21.15.7:6835/1021580938]' entity='mds.foofs.smithi007.fkkfbl' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.194:0/2491978951"}]': finished 2024-09-17T00:10:44.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:43 smithi007 ceph-mon[65487]: osdmap e159: 8 total, 8 up, 8 in 2024-09-17T00:10:45.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: pgmap v325: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 6.8 KiB/s wr, 9 op/s 2024-09-17T00:10:45.797 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.798 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:45 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: pgmap v325: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 6.8 KiB/s wr, 9 op/s 2024-09-17T00:10:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:45.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:45 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:47.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:46 smithi194 ceph-mon[51517]: pgmap v326: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-09-17T00:10:47.170 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:46 smithi007 ceph-mon[65487]: pgmap v326: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-09-17T00:10:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: pgmap v327: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.100 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi007.lardmp"}]: dispatch 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi007.lardmp"}]': finished 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.wpbzoi"}]: dispatch 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.wpbzoi"}]': finished 2024-09-17T00:10:50.101 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:49 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: pgmap v327: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.160 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi007.lardmp"}]: dispatch 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi007.lardmp"}]': finished 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.wpbzoi"}]: dispatch 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi194.wpbzoi"}]': finished 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.161 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:49 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:50.732 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:50 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all nfs 2024-09-17T00:10:50.733 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:50 smithi007 ceph-mon[65487]: Upgrade: Setting container_image for all nvmeof 2024-09-17T00:10:51.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:50 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all nfs 2024-09-17T00:10:51.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:50 smithi194 ceph-mon[51517]: Upgrade: Setting container_image for all nvmeof 2024-09-17T00:10:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:51 smithi007 ceph-mon[65487]: Upgrade: Updating grafana.smithi007 2024-09-17T00:10:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:51 smithi007 ceph-mon[65487]: Deploying daemon grafana.smithi007 on smithi007 2024-09-17T00:10:52.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:51 smithi007 ceph-mon[65487]: pgmap v328: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-09-17T00:10:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:51 smithi194 ceph-mon[51517]: Upgrade: Updating grafana.smithi007 2024-09-17T00:10:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:51 smithi194 ceph-mon[51517]: Deploying daemon grafana.smithi007 on smithi007 2024-09-17T00:10:52.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:51 smithi194 ceph-mon[51517]: pgmap v328: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.7 KiB/s rd, 1.6 KiB/s wr, 4 op/s 2024-09-17T00:10:54.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:53 smithi007 ceph-mon[65487]: pgmap v329: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 716 B/s wr, 0 op/s 2024-09-17T00:10:54.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:53 smithi194 ceph-mon[51517]: pgmap v329: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 716 B/s wr, 0 op/s 2024-09-17T00:10:55.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:55.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:10:56.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:55 smithi194 ceph-mon[51517]: pgmap v330: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 598 B/s wr, 0 op/s 2024-09-17T00:10:56.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:55 smithi007 ceph-mon[65487]: pgmap v330: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 85 B/s rd, 598 B/s wr, 0 op/s 2024-09-17T00:10:56.647 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:10:57.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:56 smithi007 ceph-mon[65487]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:57.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:56 smithi007 ceph-mon[65487]: pgmap v331: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 597 B/s wr, 0 op/s 2024-09-17T00:10:57.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:56 smithi194 ceph-mon[51517]: from='client.34544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:57.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:56 smithi194 ceph-mon[51517]: pgmap v331: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 597 B/s wr, 0 op/s 2024-09-17T00:10:57.344 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:10:57.344 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (10m) 12s ago 19m 26.5M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (6m) 12s ago 19m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (6m) 15s ago 18m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (10m) 12s ago 18m 54.3M - 8.3.5 dad864ee21e9 58b9d3e9093b 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (60s) 12s ago 13m 39.2M - 19.1.1-330-gc2ddf972 ebb0615149de 2fd65c3902d3 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (37s) 15s ago 13m 12.9M - 19.1.1-330-gc2ddf972 ebb0615149de 0868978d8471 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (9m) 12s ago 20m 510M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (8m) 15s ago 18m 443M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (8m) 12s ago 20m 62.5M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (7m) 15s ago 18m 45.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (24s) 12s ago 12m 60.1M - 5.5 ebb0615149de 7a74c2e1ce15 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (18s) 15s ago 12m 15.5M - 5.5 ebb0615149de 4830116cfb5b 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (9m) 12s ago 18m 16.1M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:10:57.345 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (9m) 15s ago 18m 16.7M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (6m) 12s ago 17m 367M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (5m) 12s ago 17m 304M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (4m) 12s ago 16m 372M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (3m) 12s ago 16m 356M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (3m) 15s ago 15m 332M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (2m) 15s ago 15m 299M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (2m) 15s ago 15m 259M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 81dd179cd962 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (87s) 15s ago 14m 312M 2704M 19.1.1-330-gc2ddf972 ebb0615149de eac97979e96c 2024-09-17T00:10:57.346 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (8m) 12s ago 18m 61.2M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T00:10:57.769 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:10:57.770 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:10:58.127 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:10:58.127 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": true, 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [ 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "mds", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "crash", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "osd", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "nfs", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "mon", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr" 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": "18/23 daemons upgraded", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "Currently upgrading grafana daemons", 2024-09-17T00:10:58.128 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:10:58.129 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:10:58.129 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:57 smithi007 ceph-mon[65487]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:58.129 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:57 smithi007 ceph-mon[65487]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:58.129 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:57 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/4097379811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:58.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:57 smithi194 ceph-mon[51517]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:57 smithi194 ceph-mon[51517]: from='client.34552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:58.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:57 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/4097379811' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:10:58.553 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:10:59.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:58 smithi007 ceph-mon[65487]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:59.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:58 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/1807335461' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:10:59.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:10:58 smithi007 ceph-mon[65487]: pgmap v332: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:10:59.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:58 smithi194 ceph-mon[51517]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:10:59.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:58 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/1807335461' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:10:59.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:10:58 smithi194 ceph-mon[51517]: pgmap v332: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:02.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:01 smithi194 ceph-mon[51517]: pgmap v333: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:02.168 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:01 smithi007 ceph-mon[65487]: pgmap v333: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:04.143 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:03 smithi007 ceph-mon[65487]: pgmap v334: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:04.143 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:03 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:04.144 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:03 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:04.144 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:03 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:04.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:03 smithi194 ceph-mon[51517]: pgmap v334: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:03 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:03 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:04.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:03 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:05.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:04 smithi194 ceph-mon[51517]: pgmap v335: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:05.159 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:04 smithi007 ceph-mon[65487]: pgmap v335: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:07 smithi194 ceph-mon[51517]: pgmap v336: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:08.249 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:07 smithi007 ceph-mon[65487]: pgmap v336: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:09.832 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.832 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.832 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:09 smithi007 ceph-mon[65487]: pgmap v337: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:09.832 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.832 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:09 smithi194 ceph-mon[51517]: pgmap v337: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:09.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:09.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:10.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:10 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:10.908 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:10 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:11.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:11 smithi007 ceph-mon[65487]: pgmap v338: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:11.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:11 smithi194 ceph-mon[51517]: pgmap v338: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:14.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: pgmap v339: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:14.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:11:14.097 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.098 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T00:11:14.099 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.100 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T00:11:14.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T00:11:14.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T00:11:14.101 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T00:11:14.102 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.103 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: pgmap v339: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.104 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-17T00:11:14.105 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-17T00:11:14.106 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:14.108 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:14 smithi007 ceph-mon[65487]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:11:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:14 smithi007 ceph-mon[65487]: Upgrade: Finalizing container_image settings 2024-09-17T00:11:15.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:14 smithi007 ceph-mon[65487]: Upgrade: Complete! 2024-09-17T00:11:15.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:14 smithi194 ceph-mon[51517]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-17T00:11:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:14 smithi194 ceph-mon[51517]: Upgrade: Finalizing container_image settings 2024-09-17T00:11:15.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:14 smithi194 ceph-mon[51517]: Upgrade: Complete! 2024-09-17T00:11:16.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:15 smithi007 ceph-mon[65487]: pgmap v340: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:16.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:15 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:15 smithi194 ceph-mon[51517]: pgmap v340: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:16.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:15 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:11:18.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:17 smithi007 ceph-mon[65487]: pgmap v341: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:18.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:17 smithi194 ceph-mon[51517]: pgmap v341: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:20.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:19 smithi007 ceph-mon[65487]: pgmap v342: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:20.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:19 smithi194 ceph-mon[51517]: pgmap v342: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:22.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:21 smithi007 ceph-mon[65487]: pgmap v343: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:22.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:21 smithi194 ceph-mon[51517]: pgmap v343: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:24.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:23 smithi007 ceph-mon[65487]: pgmap v344: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:24.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:23 smithi194 ceph-mon[51517]: pgmap v344: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:25.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:25.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:26.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:25 smithi007 ceph-mon[65487]: pgmap v345: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:26.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:25 smithi194 ceph-mon[51517]: pgmap v345: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:28.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:27 smithi007 ceph-mon[65487]: pgmap v346: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:28.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:27 smithi194 ceph-mon[51517]: pgmap v346: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:29.614 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T00:11:30.107 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:29 smithi007 ceph-mon[65487]: pgmap v347: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:30.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:29 smithi194 ceph-mon[51517]: pgmap v347: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:30.948 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:30 smithi007 ceph-mon[65487]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:11:31.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:30 smithi194 ceph-mon[51517]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:11:31.724 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (10m) 23s ago 19m 26.5M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (7m) 23s ago 19m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (7m) 49s ago 19m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (28s) 23s ago 19m 46.7M - 9.4.12 2bacad6d85d8 fcdfcd15aa80 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (94s) 23s ago 13m 39.2M - 19.1.1-330-gc2ddf972 ebb0615149de 2fd65c3902d3 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (71s) 49s ago 13m 12.9M - 19.1.1-330-gc2ddf972 ebb0615149de 0868978d8471 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (9m) 23s ago 20m 517M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (9m) 49s ago 19m 443M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (8m) 23s ago 21m 63.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (8m) 49s ago 19m 45.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:11:31.725 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (58s) 23s ago 13m 60.4M - 5.5 ebb0615149de 7a74c2e1ce15 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (53s) 49s ago 13m 15.5M - 5.5 ebb0615149de 4830116cfb5b 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (10m) 23s ago 19m 16.0M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (10m) 49s ago 18m 16.7M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (6m) 23s ago 18m 368M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (6m) 23s ago 17m 305M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (5m) 23s ago 17m 372M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (4m) 23s ago 16m 357M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (3m) 49s ago 16m 332M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (3m) 49s ago 16m 299M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (2m) 49s ago 15m 259M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 81dd179cd962 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (2m) 49s ago 15m 312M 2704M 19.1.1-330-gc2ddf972 ebb0615149de eac97979e96c 2024-09-17T00:11:31.726 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (9m) 23s ago 18m 63.7M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:11:32.056 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:31 smithi007 ceph-mon[65487]: pgmap v348: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:32.057 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:31 smithi007 ceph-mon[65487]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:11:32.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:31 smithi194 ceph-mon[51517]: pgmap v348: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:32.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:31 smithi194 ceph-mon[51517]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:11:32.387 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T00:11:33.044 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:32 smithi007 ceph-mon[65487]: pgmap v349: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:33.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:32 smithi194 ceph-mon[51517]: pgmap v349: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:34.549 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:11:34.550 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:11:34.551 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T00:11:34.551 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:11:34.551 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:11:34.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:34 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3341440084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:34.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:34 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3341440084' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:11:35.203 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-17T00:11:35.800 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:35 smithi007 ceph-mon[65487]: pgmap v350: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:35.909 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:35 smithi194 ceph-mon[51517]: pgmap v350: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:37.116 INFO:teuthology.orchestra.run.smithi007.stdout:wait for servicemap items w/ changing names to refresh 2024-09-17T00:11:37.116 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:36 smithi007 ceph-mon[65487]: pgmap v351: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:37.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:36 smithi194 ceph-mon[51517]: pgmap v351: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:37.751 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'sleep 60' 2024-09-17T00:11:39.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:38 smithi194 ceph-mon[51517]: pgmap v352: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:39.177 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:38 smithi007 ceph-mon[65487]: pgmap v352: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:40.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:40.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:41.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:40 smithi007 ceph-mon[65487]: pgmap v353: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:41.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:40 smithi194 ceph-mon[51517]: pgmap v353: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:43.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:42 smithi007 ceph-mon[65487]: pgmap v354: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:43.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:42 smithi194 ceph-mon[51517]: pgmap v354: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:45.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:44 smithi007 ceph-mon[65487]: pgmap v355: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:44 smithi194 ceph-mon[51517]: pgmap v355: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:47.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:46 smithi194 ceph-mon[51517]: pgmap v356: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:47.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:46 smithi007 ceph-mon[65487]: pgmap v356: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:49.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:48 smithi194 ceph-mon[51517]: pgmap v357: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:49.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:48 smithi007 ceph-mon[65487]: pgmap v357: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:51.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:50 smithi007 ceph-mon[65487]: pgmap v358: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:51.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:50 smithi194 ceph-mon[51517]: pgmap v358: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:53.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:52 smithi194 ceph-mon[51517]: pgmap v359: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:53.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:52 smithi007 ceph-mon[65487]: pgmap v359: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:55.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:55.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:11:56.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:55 smithi007 ceph-mon[65487]: pgmap v360: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:56.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:55 smithi194 ceph-mon[51517]: pgmap v360: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:57.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:56 smithi194 ceph-mon[51517]: pgmap v361: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:57.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:56 smithi007 ceph-mon[65487]: pgmap v361: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:11:59.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:11:58 smithi007 ceph-mon[65487]: pgmap v362: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:11:59.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:11:58 smithi194 ceph-mon[51517]: pgmap v362: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:01.211 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:00 smithi194 ceph-mon[51517]: pgmap v363: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:01.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:00 smithi007 ceph-mon[65487]: pgmap v363: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:03.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:02 smithi007 ceph-mon[65487]: pgmap v364: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:03.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:02 smithi194 ceph-mon[51517]: pgmap v364: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:05.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:04 smithi007 ceph-mon[65487]: pgmap v365: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:05.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:04 smithi194 ceph-mon[51517]: pgmap v365: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:07.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:06 smithi007 ceph-mon[65487]: pgmap v366: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:07.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:06 smithi194 ceph-mon[51517]: pgmap v366: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:09.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:08 smithi007 ceph-mon[65487]: pgmap v367: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:09.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:08 smithi194 ceph-mon[51517]: pgmap v367: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:10.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:10.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:11.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:10 smithi007 ceph-mon[65487]: pgmap v368: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:11.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:10 smithi194 ceph-mon[51517]: pgmap v368: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:13.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:12 smithi007 ceph-mon[65487]: pgmap v369: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:13.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:12 smithi194 ceph-mon[51517]: pgmap v369: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:12:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:14.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:13 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-17T00:12:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:14.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:13 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:15.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:14 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:12:15.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:14 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:12:15.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:14 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:15.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:14 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:15.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:14 smithi007 ceph-mon[65487]: pgmap v370: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:15.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:14 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-17T00:12:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:14 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-17T00:12:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:14 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:14 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' 2024-09-17T00:12:15.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:14 smithi194 ceph-mon[51517]: pgmap v370: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:17.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:16 smithi007 ceph-mon[65487]: pgmap v371: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:17.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:16 smithi194 ceph-mon[51517]: pgmap v371: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:19.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:18 smithi007 ceph-mon[65487]: pgmap v372: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:19.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:18 smithi194 ceph-mon[51517]: pgmap v372: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:21.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:20 smithi007 ceph-mon[65487]: pgmap v373: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:21.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:20 smithi194 ceph-mon[51517]: pgmap v373: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:23.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:22 smithi007 ceph-mon[65487]: pgmap v374: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:23.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:22 smithi194 ceph-mon[51517]: pgmap v374: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:25.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:24 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:25.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:24 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:26.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:25 smithi007 ceph-mon[65487]: pgmap v375: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:26.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:25 smithi194 ceph-mon[51517]: pgmap v375: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:27.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:26 smithi007 ceph-mon[65487]: pgmap v376: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:27.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:26 smithi194 ceph-mon[51517]: pgmap v376: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:29.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:28 smithi007 ceph-mon[65487]: pgmap v377: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:29.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:28 smithi194 ceph-mon[51517]: pgmap v377: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:31.211 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:30 smithi194 ceph-mon[51517]: pgmap v378: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:31.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:30 smithi007 ceph-mon[65487]: pgmap v378: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:33.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:32 smithi007 ceph-mon[65487]: pgmap v379: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:33.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:32 smithi194 ceph-mon[51517]: pgmap v379: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:35.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:34 smithi007 ceph-mon[65487]: pgmap v380: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:35.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:34 smithi194 ceph-mon[51517]: pgmap v380: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:37.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:36 smithi007 ceph-mon[65487]: pgmap v381: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:37.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:36 smithi194 ceph-mon[51517]: pgmap v381: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:39.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:38 smithi007 ceph-mon[65487]: pgmap v382: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:39.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:38 smithi194 ceph-mon[51517]: pgmap v382: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:40.264 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:39 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:40.295 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch ps' 2024-09-17T00:12:40.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:39 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:41.280 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:40 smithi007 ceph-mon[65487]: pgmap v383: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:41.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:40 smithi194 ceph-mon[51517]: pgmap v383: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:alertmanager.smithi007 smithi007 *:9093,9094 running (11m) 93s ago 20m 26.5M - 0.25.0 c8568f914cd2 5ed6c9f4504c 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi007 smithi007 running (8m) 93s ago 20m 6891k - 19.1.1-330-gc2ddf972 ebb0615149de d5fbf86ab913 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:crash.smithi194 smithi194 running (8m) 2m ago 20m 6895k - 19.1.1-330-gc2ddf972 ebb0615149de 47b6f7458703 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:grafana.smithi007 smithi007 *:3000 running (99s) 93s ago 20m 46.7M - 9.4.12 2bacad6d85d8 fcdfcd15aa80 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi007.fkkfbl smithi007 running (2m) 93s ago 15m 39.2M - 19.1.1-330-gc2ddf972 ebb0615149de 2fd65c3902d3 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:mds.foofs.smithi194.emredf smithi194 running (2m) 2m ago 14m 12.9M - 19.1.1-330-gc2ddf972 ebb0615149de 0868978d8471 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi007.czdphd smithi007 *:8443,9283,8765 running (11m) 93s ago 22m 517M - 19.1.1-330-gc2ddf972 ebb0615149de 999fa851b882 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:mgr.smithi194.xlteft smithi194 *:8443,9283,8765 running (10m) 2m ago 20m 443M - 19.1.1-330-gc2ddf972 ebb0615149de bc64638e44fb 2024-09-17T00:12:42.241 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi007 smithi007 running (9m) 93s ago 22m 63.6M 2048M 19.1.1-330-gc2ddf972 ebb0615149de b047b28b4eda 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:mon.smithi194 smithi194 running (9m) 2m ago 20m 45.4M 2048M 19.1.1-330-gc2ddf972 ebb0615149de 3c19a28e308b 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.0.0.smithi007.lardmp smithi007 *:2049 running (2m) 93s ago 14m 60.4M - 5.5 ebb0615149de 7a74c2e1ce15 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:nfs.foo.1.0.smithi194.wpbzoi smithi194 *:2049 running (2m) 2m ago 14m 15.5M - 5.5 ebb0615149de 4830116cfb5b 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi007 smithi007 *:9100 running (11m) 93s ago 20m 16.0M - 1.5.0 0da6a335fe13 fb5aa5691ed7 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:node-exporter.smithi194 smithi194 *:9100 running (11m) 2m ago 20m 16.7M - 1.5.0 0da6a335fe13 6a742d47ce39 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:osd.0 smithi007 running (8m) 93s ago 19m 368M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 23ad063d06bd 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:osd.1 smithi007 running (7m) 93s ago 18m 305M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 7e9658d78e9c 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:osd.2 smithi007 running (6m) 93s ago 18m 372M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 5cfd3d046e11 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:osd.3 smithi007 running (5m) 93s ago 18m 357M 1937M 19.1.1-330-gc2ddf972 ebb0615149de 6956484a2f38 2024-09-17T00:12:42.242 INFO:teuthology.orchestra.run.smithi007.stdout:osd.4 smithi194 running (5m) 2m ago 17m 332M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 508bdb2e794c 2024-09-17T00:12:42.243 INFO:teuthology.orchestra.run.smithi007.stdout:osd.5 smithi194 running (4m) 2m ago 17m 299M 2704M 19.1.1-330-gc2ddf972 ebb0615149de cd9a4795f158 2024-09-17T00:12:42.243 INFO:teuthology.orchestra.run.smithi007.stdout:osd.6 smithi194 running (3m) 2m ago 16m 259M 2704M 19.1.1-330-gc2ddf972 ebb0615149de 81dd179cd962 2024-09-17T00:12:42.243 INFO:teuthology.orchestra.run.smithi007.stdout:osd.7 smithi194 running (3m) 2m ago 16m 312M 2704M 19.1.1-330-gc2ddf972 ebb0615149de eac97979e96c 2024-09-17T00:12:42.243 INFO:teuthology.orchestra.run.smithi007.stdout:prometheus.smithi007 smithi007 *:9095 running (10m) 93s ago 19m 63.7M - 2.43.0 a07b618ecd1d 49c0b5f35d82 2024-09-17T00:12:42.897 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph orch upgrade status' 2024-09-17T00:12:43.206 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:42 smithi007 ceph-mon[65487]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:12:43.206 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:42 smithi007 ceph-mon[65487]: pgmap v384: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:43.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:42 smithi194 ceph-mon[51517]: from='client.34580 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:12:43.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:42 smithi194 ceph-mon[51517]: pgmap v384: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:44.855 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:12:44.855 INFO:teuthology.orchestra.run.smithi007.stdout: "target_image": null, 2024-09-17T00:12:44.855 INFO:teuthology.orchestra.run.smithi007.stdout: "in_progress": false, 2024-09-17T00:12:44.855 INFO:teuthology.orchestra.run.smithi007.stdout: "which": "", 2024-09-17T00:12:44.855 INFO:teuthology.orchestra.run.smithi007.stdout: "services_complete": [], 2024-09-17T00:12:44.855 INFO:teuthology.orchestra.run.smithi007.stdout: "progress": null, 2024-09-17T00:12:44.856 INFO:teuthology.orchestra.run.smithi007.stdout: "message": "", 2024-09-17T00:12:44.856 INFO:teuthology.orchestra.run.smithi007.stdout: "is_paused": false 2024-09-17T00:12:44.856 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:12:45.240 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:44 smithi007 ceph-mon[65487]: pgmap v385: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:45.240 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:44 smithi007 ceph-mon[65487]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:12:45.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:44 smithi194 ceph-mon[51517]: pgmap v385: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:45.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:44 smithi194 ceph-mon[51517]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:12:45.469 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph health detail' 2024-09-17T00:12:46.982 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:46 smithi007 ceph-mon[65487]: pgmap v386: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:47.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:46 smithi194 ceph-mon[51517]: pgmap v386: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:47.521 INFO:teuthology.orchestra.run.smithi007.stdout:HEALTH_OK 2024-09-17T00:12:48.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:47 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/1667425499' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:12:48.136 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions' 2024-09-17T00:12:48.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:47 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/1667425499' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-17T00:12:49.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:48 smithi007 ceph-mon[65487]: pgmap v387: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:49.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:48 smithi194 ceph-mon[51517]: pgmap v387: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:50.311 INFO:teuthology.orchestra.run.smithi007.stdout:{ 2024-09-17T00:12:50.312 INFO:teuthology.orchestra.run.smithi007.stdout: "mon": { 2024-09-17T00:12:50.312 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:12:50.312 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:12:50.312 INFO:teuthology.orchestra.run.smithi007.stdout: "mgr": { 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "osd": { 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 8 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "mds": { 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 2 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "overall": { 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)": 14 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:12:50.313 INFO:teuthology.orchestra.run.smithi007.stdout:} 2024-09-17T00:12:50.598 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:50 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3130867374' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:12:50.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:50 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3130867374' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:12:51.021 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-17T00:12:51.586 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:51 smithi007 ceph-mon[65487]: pgmap v388: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:51.658 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:51 smithi194 ceph-mon[51517]: pgmap v388: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:53.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:52 smithi007 ceph-mon[65487]: pgmap v389: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:53.194 INFO:teuthology.orchestra.run.smithi007.stdout:true 2024-09-17T00:12:53.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:52 smithi194 ceph-mon[51517]: pgmap v389: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:53.970 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -e sha1=c2ddf9721ef254645201cdbabd9c8f49c30a586f -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-17T00:12:54.281 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:53 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3779881668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:12:54.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:53 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3779881668' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:12:55.263 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:54 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:55.263 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:54 smithi007 ceph-mon[65487]: pgmap v390: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:55.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:54 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:12:55.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:54 smithi194 ceph-mon[51517]: pgmap v390: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:56.087 INFO:teuthology.orchestra.run.smithi007.stdout: "ceph version 19.1.1-330-gc2ddf972 (c2ddf9721ef254645201cdbabd9c8f49c30a586f) squid (stable)" 2024-09-17T00:12:56.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:56 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/3819460397' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:12:56.409 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:56 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/3819460397' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-17T00:12:56.679 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-17T00:12:56.681 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-17T00:12:56.681 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-17T00:12:57.316 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:57 smithi007 ceph-mon[65487]: pgmap v391: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:57.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:57 smithi194 ceph-mon[51517]: pgmap v391: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-17T00:12:58.713 INFO:teuthology.orchestra.run.smithi007.stdout: 2024-09-17T00:12:58.714 INFO:teuthology.orchestra.run.smithi007.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-16T23:51:09.466050Z", "last_refresh": "2024-09-17T00:11:08.574906Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-16T23:51:03.946294Z", "last_refresh": "2024-09-17T00:10:41.793718Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-16T23:51:06.592239Z", "last_refresh": "2024-09-17T00:11:08.575051Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-16T23:57:36.943452Z", "last_refresh": "2024-09-17T00:10:41.794308Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-16T23:51:01.993043Z", "last_refresh": "2024-09-17T00:10:41.793821Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi007:172.21.15.7=smithi007", "smithi194:172.21.15.194=smithi194"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-16T23:51:53.906495Z", "last_refresh": "2024-09-17T00:10:41.793896Z", "running": 2, "size": 2}}, {"events": ["2024-09-17T00:12:14.024385Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-16T23:57:58.547749Z", "last_refresh": "2024-09-17T00:10:41.794377Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-16T23:51:08.211904Z", "last_refresh": "2024-09-17T00:10:41.793965Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3", "last_refresh": "2024-09-17T00:10:41.794035Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-16T23:51:05.157862Z", "last_refresh": "2024-09-17T00:11:08.575190Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-17T00:12:59.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:58 smithi007 ceph-mon[65487]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T00:12:59.013 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:12:58 smithi007 ceph-mon[65487]: pgmap v392: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:12:59.336 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-17T00:12:59.337 DEBUG:teuthology.parallel:result is None 2024-09-17T00:12:59.337 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-17T00:12:59.352 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-17T00:12:59.352 DEBUG:teuthology.orchestra.run.smithi007:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-17T00:12:59.379 INFO:teuthology.orchestra.run.smithi007.stderr:+ umount /mnt/foo 2024-09-17T00:12:59.400 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T00:12:59.409 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-17T00:12:59.410 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-17T00:12:59.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:58 smithi194 ceph-mon[51517]: from='client.34604 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-17T00:12:59.410 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:12:58 smithi194 ceph-mon[51517]: pgmap v392: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:01.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:00 smithi007 ceph-mon[65487]: pgmap v393: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:01.210 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:00 smithi194 ceph-mon[51517]: pgmap v393: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:01.656 INFO:teuthology.orchestra.run.smithi007.stdout: "foo" 2024-09-17T00:13:02.235 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:01 smithi007 ceph-mon[65487]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:13:02.259 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-17T00:13:02.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:01 smithi194 ceph-mon[51517]: from='client.34608 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:13:03.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:02 smithi007 ceph-mon[65487]: pgmap v394: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:03.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:02 smithi194 ceph-mon[51517]: pgmap v394: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:04.260 INFO:teuthology.orchestra.run.smithi007.stdout:[ 2024-09-17T00:13:04.260 INFO:teuthology.orchestra.run.smithi007.stdout: { 2024-09-17T00:13:04.260 INFO:teuthology.orchestra.run.smithi007.stdout: "access_type": "RW", 2024-09-17T00:13:04.260 INFO:teuthology.orchestra.run.smithi007.stdout: "clients": [], 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "cluster_id": "foo", 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "export_id": 1, 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "fsal": { 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "fs_name": "foofs", 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "name": "CEPH", 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "user_id": "nfs.foo.1" 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: }, 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "path": "/", 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "protocols": [ 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: 4 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: ], 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "pseudo": "/fake", 2024-09-17T00:13:04.261 INFO:teuthology.orchestra.run.smithi007.stdout: "security_label": true, 2024-09-17T00:13:04.262 INFO:teuthology.orchestra.run.smithi007.stdout: "squash": "none", 2024-09-17T00:13:04.262 INFO:teuthology.orchestra.run.smithi007.stdout: "transports": [ 2024-09-17T00:13:04.262 INFO:teuthology.orchestra.run.smithi007.stdout: "TCP" 2024-09-17T00:13:04.262 INFO:teuthology.orchestra.run.smithi007.stdout: ] 2024-09-17T00:13:04.262 INFO:teuthology.orchestra.run.smithi007.stdout: } 2024-09-17T00:13:04.262 INFO:teuthology.orchestra.run.smithi007.stdout:] 2024-09-17T00:13:04.882 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-17T00:13:06.098 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:05 smithi007 ceph-mon[65487]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:13:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:05 smithi007 ceph-mon[65487]: pgmap v395: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:06.099 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:05 smithi007 ceph-mon[65487]: mgrmap e44: smithi007.czdphd(active, since 10m), standbys: smithi194.xlteft 2024-09-17T00:13:06.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:05 smithi194 ceph-mon[51517]: from='client.34612 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-17T00:13:06.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:05 smithi194 ceph-mon[51517]: pgmap v395: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-17T00:13:06.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:05 smithi194 ceph-mon[51517]: mgrmap e44: smithi007.czdphd(active, since 10m), standbys: smithi194.xlteft 2024-09-17T00:13:06.760 INFO:teuthology.orchestra.run.smithi007.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-17T00:13:06.760 INFO:teuthology.orchestra.run.smithi007.stdout:foo grace 2024-09-17T00:13:06.760 INFO:teuthology.orchestra.run.smithi007.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-17T00:13:06.760 INFO:teuthology.orchestra.run.smithi007.stdout:foo conf-nfs.foo 2024-09-17T00:13:06.760 INFO:teuthology.orchestra.run.smithi007.stdout:foo export-1 2024-09-17T00:13:07.370 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-17T00:13:07.386 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi007.front.sepia.ceph.com 2024-09-17T00:13:07.386 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-17T00:13:07.386 DEBUG:teuthology.orchestra.run.smithi007:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-17T00:13:07.387 DEBUG:teuthology.orchestra.run.smithi007:> ' 2024-09-17T00:13:08.008 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:07 smithi007 ceph-mon[65487]: pgmap v396: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T00:13:08.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:07 smithi194 ceph-mon[51517]: pgmap v396: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-09-17T00:13:09.098 INFO:teuthology.orchestra.run.smithi007.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-17T00:13:09.348 INFO:teuthology.orchestra.run.smithi007.stderr:+ [[ 7 -gt 2 ]] 2024-09-17T00:13:09.928 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:09 smithi007 ceph-mon[65487]: pgmap v397: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T00:13:09.928 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:09 smithi007 ceph-mon[65487]: from='client.? 172.21.15.7:0/60850576' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-17T00:13:09.928 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:09 smithi007 ceph-mon[65487]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:13:09.951 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-17T00:13:09.960 INFO:tasks.cephadm:Teardown begin 2024-09-17T00:13:09.961 DEBUG:teuthology.orchestra.run.smithi007:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:13:09.989 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:13:10.023 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-17T00:13:10.023 DEBUG:teuthology.orchestra.run.smithi007:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T00:13:10.045 DEBUG:teuthology.orchestra.run.smithi194:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-17T00:13:10.081 INFO:tasks.cephadm:Stopping all daemons... 2024-09-17T00:13:10.082 INFO:tasks.cephadm.mon.smithi007:Stopping mon.smithi007... 2024-09-17T00:13:10.082 DEBUG:teuthology.orchestra.run.smithi007:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007 2024-09-17T00:13:10.158 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:09 smithi194 ceph-mon[51517]: pgmap v397: 97 pgs: 97 active+clean; 214 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-17T00:13:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:09 smithi194 ceph-mon[51517]: from='client.? 172.21.15.7:0/60850576' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"}]: dispatch 2024-09-17T00:13:10.159 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:09 smithi194 ceph-mon[51517]: from='mgr.34100 172.21.15.7:0/1243885051' entity='mgr.smithi007.czdphd' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-17T00:13:10.349 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:10 smithi007 systemd[1]: Stopping Ceph mon.smithi007 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:10.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[65483]: 2024-09-17T00:13:10.384+0000 7f23fa4e9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi007 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:10.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:10 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007[65483]: 2024-09-17T00:13:10.384+0000 7f23fa4e9640 -1 mon.smithi007@0(leader) e3 *** Got Signal Terminated *** 2024-09-17T00:13:10.849 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:10 smithi007 podman[98201]: 2024-09-17 00:13:10.619680044 +0000 UTC m=+0.339871203 container died b047b28b4edac807811478889d46e09a66d18bd77a524e9e006fe6b656bcf349 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:13:11.327 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:11 smithi007 podman[98201]: 2024-09-17 00:13:11.009009608 +0000 UTC m=+0.729200766 container cleanup b047b28b4edac807811478889d46e09a66d18bd77a524e9e006fe6b656bcf349 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:13:11.327 INFO:journalctl@ceph.mon.smithi007.smithi007.stdout:Sep 17 00:13:11 smithi007 bash[98201]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi007 2024-09-17T00:13:11.502 DEBUG:teuthology.orchestra.run.smithi007:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi007.service' 2024-09-17T00:13:11.551 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:11.551 INFO:tasks.cephadm.mon.smithi007:Stopped mon.smithi007 2024-09-17T00:13:11.552 INFO:tasks.cephadm.mon.smithi194:Stopping mon.smithi194... 2024-09-17T00:13:11.552 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi194 2024-09-17T00:13:11.846 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:11 smithi194 systemd[1]: Stopping Ceph mon.smithi194 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:12.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194[51513]: 2024-09-17T00:13:11.848+0000 7fdb85b69640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi194 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:12.111 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194[51513]: 2024-09-17T00:13:11.848+0000 7fdb85b69640 -1 mon.smithi194@1(peon) e3 *** Got Signal Terminated *** 2024-09-17T00:13:12.408 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:12 smithi194 podman[72461]: 2024-09-17 00:13:12.113916794 +0000 UTC m=+0.362945840 container died 3c19a28e308b78aebe102cc99f9fbc5e1451ec743d302ee30dfa7a13fec6f1b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-17T00:13:12.856 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:12 smithi194 podman[72461]: 2024-09-17 00:13:12.631413508 +0000 UTC m=+0.880442545 container cleanup 3c19a28e308b78aebe102cc99f9fbc5e1451ec743d302ee30dfa7a13fec6f1b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0) 2024-09-17T00:13:12.856 INFO:journalctl@ceph.mon.smithi194.smithi194.stdout:Sep 17 00:13:12 smithi194 bash[72461]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-mon-smithi194 2024-09-17T00:13:13.078 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@mon.smithi194.service' 2024-09-17T00:13:13.128 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:13.128 INFO:tasks.cephadm.mon.smithi194:Stopped mon.smithi194 2024-09-17T00:13:13.128 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-17T00:13:13.128 DEBUG:teuthology.orchestra.run.smithi007:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.0 2024-09-17T00:13:13.422 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:13 smithi007 systemd[1]: Stopping Ceph osd.0 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:13.849 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:13 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:13:13.454+0000 7fbf61aa1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:13.849 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:13 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:13:13.454+0000 7fbf61aa1640 -1 osd.0 159 *** Got signal Terminated *** 2024-09-17T00:13:13.849 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:13 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0[71435]: 2024-09-17T00:13:13.454+0000 7fbf61aa1640 -1 osd.0 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:13:19.345 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:19 smithi007 podman[98334]: 2024-09-17 00:13:19.042047121 +0000 UTC m=+5.714452794 container died 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-17T00:13:19.599 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:19 smithi007 podman[98334]: 2024-09-17 00:13:19.438418971 +0000 UTC m=+6.110824637 container cleanup 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2) 2024-09-17T00:13:19.599 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:19 smithi007 bash[98334]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0 2024-09-17T00:13:20.033 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:19 smithi007 podman[98349]: 2024-09-17 00:13:19.706625857 +0000 UTC m=+0.665742806 container remove 23ad063d06bd233eb99b9dd67ca8206c35dd17852fb45d2ba5263d454bef69f6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-17T00:13:20.307 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:20 smithi007 podman[98438]: 2024-09-17 00:13:19.939125735 +0000 UTC m=+0.016719284 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:13:20.307 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:20 smithi007 podman[98438]: 2024-09-17 00:13:20.068048158 +0000 UTC m=+0.145641697 container create 083d842dcd2faeec4b50ac928b126a9e1d51882b9f4a9e2724a1f854252ccd2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:13:20.307 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:20 smithi007 podman[98438]: 2024-09-17 00:13:20.31028769 +0000 UTC m=+0.387881238 container init 083d842dcd2faeec4b50ac928b126a9e1d51882b9f4a9e2724a1f854252ccd2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-17T00:13:20.599 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:20 smithi007 podman[98438]: 2024-09-17 00:13:20.313979016 +0000 UTC m=+0.391572555 container start 083d842dcd2faeec4b50ac928b126a9e1d51882b9f4a9e2724a1f854252ccd2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-17T00:13:20.599 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:20 smithi007 podman[98438]: 2024-09-17 00:13:20.393405507 +0000 UTC m=+0.470999066 container attach 083d842dcd2faeec4b50ac928b126a9e1d51882b9f4a9e2724a1f854252ccd2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-09-17T00:13:20.599 INFO:journalctl@ceph.osd.0.smithi007.stdout:Sep 17 00:13:20 smithi007 podman[98438]: 2024-09-17 00:13:20.514126207 +0000 UTC m=+0.591719745 container died 083d842dcd2faeec4b50ac928b126a9e1d51882b9f4a9e2724a1f854252ccd2f (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T00:13:21.178 DEBUG:teuthology.orchestra.run.smithi007:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.0.service' 2024-09-17T00:13:21.220 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:21.220 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-17T00:13:21.221 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-17T00:13:21.221 DEBUG:teuthology.orchestra.run.smithi007:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.1 2024-09-17T00:13:21.582 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:21 smithi007 systemd[1]: Stopping Ceph osd.1 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:21.849 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:13:21.584+0000 7f576e21a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:21.849 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:13:21.584+0000 7f576e21a640 -1 osd.1 159 *** Got signal Terminated *** 2024-09-17T00:13:21.849 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:21 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1[74025]: 2024-09-17T00:13:21.584+0000 7f576e21a640 -1 osd.1 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:13:27.457 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:27 smithi007 podman[98539]: 2024-09-17 00:13:27.148853638 +0000 UTC m=+5.686810385 container died 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T00:13:27.782 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:27 smithi007 podman[98539]: 2024-09-17 00:13:27.533265502 +0000 UTC m=+6.071222240 container cleanup 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-17T00:13:27.782 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:27 smithi007 bash[98539]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1 2024-09-17T00:13:28.098 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:27 smithi007 podman[98587]: 2024-09-17 00:13:27.785079242 +0000 UTC m=+0.631130044 container remove 7e9658d78e9c752dcd20ffdcb25ec505745ffb14f56bde8744c49765c2f2c495 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T00:13:28.385 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:28 smithi007 podman[98678]: 2024-09-17 00:13:28.011055601 +0000 UTC m=+0.016904216 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:13:28.385 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:28 smithi007 podman[98678]: 2024-09-17 00:13:28.110933073 +0000 UTC m=+0.116781714 container create b1690e5029936d42f062ba42d4af2e2024965343a857e823a7348707feeb87cc (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-17T00:13:28.385 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:28 smithi007 podman[98678]: 2024-09-17 00:13:28.313228469 +0000 UTC m=+0.319077081 container init b1690e5029936d42f062ba42d4af2e2024965343a857e823a7348707feeb87cc (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-17T00:13:28.385 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:28 smithi007 podman[98678]: 2024-09-17 00:13:28.316680727 +0000 UTC m=+0.322529328 container start b1690e5029936d42f062ba42d4af2e2024965343a857e823a7348707feeb87cc (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-17T00:13:28.386 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:28 smithi007 podman[98678]: 2024-09-17 00:13:28.388186666 +0000 UTC m=+0.394035279 container attach b1690e5029936d42f062ba42d4af2e2024965343a857e823a7348707feeb87cc (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:13:28.822 INFO:journalctl@ceph.osd.1.smithi007.stdout:Sep 17 00:13:28 smithi007 podman[98678]: 2024-09-17 00:13:28.522979002 +0000 UTC m=+0.528827616 container died b1690e5029936d42f062ba42d4af2e2024965343a857e823a7348707feeb87cc (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97) 2024-09-17T00:13:29.123 DEBUG:teuthology.orchestra.run.smithi007:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.1.service' 2024-09-17T00:13:29.164 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:29.165 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-17T00:13:29.165 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-17T00:13:29.165 DEBUG:teuthology.orchestra.run.smithi007:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.2 2024-09-17T00:13:29.349 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:29 smithi007 systemd[1]: Stopping Ceph osd.2 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:29.849 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:29 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:13:29.495+0000 7f13ae617640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:29.849 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:29 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:13:29.495+0000 7f13ae617640 -1 osd.2 159 *** Got signal Terminated *** 2024-09-17T00:13:29.849 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:29 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2[76791]: 2024-09-17T00:13:29.495+0000 7f13ae617640 -1 osd.2 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:13:35.169 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:34 smithi007 podman[98780]: 2024-09-17 00:13:34.891410474 +0000 UTC m=+5.488924690 container died 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-17T00:13:35.526 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:35 smithi007 podman[98780]: 2024-09-17 00:13:35.253171234 +0000 UTC m=+5.850685440 container cleanup 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, RELEASE=squid-c2ddf97, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T00:13:35.527 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:35 smithi007 bash[98780]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2 2024-09-17T00:13:35.849 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:35 smithi007 podman[98795]: 2024-09-17 00:13:35.52935479 +0000 UTC m=+0.633851134 container remove 5cfd3d046e118fa91b12c443cb1ef47b027589c69e2557585a1be32559c42a79 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T00:13:36.180 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:35 smithi007 podman[98884]: 2024-09-17 00:13:35.780112913 +0000 UTC m=+0.017399507 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:13:36.180 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:35 smithi007 podman[98884]: 2024-09-17 00:13:35.880259439 +0000 UTC m=+0.117546066 container create 3482225ba85d36d43389f6e8b2d08568c807cb7f8f3d68e14ca94928a0df82e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-17T00:13:36.180 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:36 smithi007 podman[98884]: 2024-09-17 00:13:36.107575278 +0000 UTC m=+0.344861864 container init 3482225ba85d36d43389f6e8b2d08568c807cb7f8f3d68e14ca94928a0df82e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-17T00:13:36.180 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:36 smithi007 podman[98884]: 2024-09-17 00:13:36.111023176 +0000 UTC m=+0.348309759 container start 3482225ba85d36d43389f6e8b2d08568c807cb7f8f3d68e14ca94928a0df82e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:13:36.181 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:36 smithi007 podman[98884]: 2024-09-17 00:13:36.182583401 +0000 UTC m=+0.419870010 container attach 3482225ba85d36d43389f6e8b2d08568c807cb7f8f3d68e14ca94928a0df82e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:13:36.599 INFO:journalctl@ceph.osd.2.smithi007.stdout:Sep 17 00:13:36 smithi007 podman[98884]: 2024-09-17 00:13:36.303714388 +0000 UTC m=+0.541000973 container died 3482225ba85d36d43389f6e8b2d08568c807cb7f8f3d68e14ca94928a0df82e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-17T00:13:37.005 DEBUG:teuthology.orchestra.run.smithi007:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.2.service' 2024-09-17T00:13:37.047 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:37.047 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-17T00:13:37.047 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-17T00:13:37.048 DEBUG:teuthology.orchestra.run.smithi007:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.3 2024-09-17T00:13:37.349 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:37 smithi007 systemd[1]: Stopping Ceph osd.3 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:37.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:37 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:13:37.372+0000 7f23803cf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:37.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:37 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:13:37.372+0000 7f23803cf640 -1 osd.3 159 *** Got signal Terminated *** 2024-09-17T00:13:37.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:37 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:13:37.372+0000 7f23803cf640 -1 osd.3 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:13:40.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:39 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:39.949+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:41.158 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:40.924+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:41.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:40 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:40.936+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:41.575 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:41 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:41.179+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:42.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:41 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:41.884+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:42.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:41 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:41.926+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:42.517 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:42 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:42.203+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:42.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:42 smithi007 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3[79364]: 2024-09-17T00:13:42.351+0000 7f237c9e5640 -1 osd.3 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.898546+0000 front 2024-09-17T00:13:17.898697+0000 (oldest deadline 2024-09-17T00:13:41.998633+0000) 2024-09-17T00:13:42.897 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:42 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:42.519+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:43.158 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:42 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:42.899+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:43.158 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:42 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:42.972+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:43.254 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:42 smithi007 podman[98985]: 2024-09-17 00:13:42.964346593 +0000 UTC m=+5.682390314 container died 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:13:43.504 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:43 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:43.253+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:43.520 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:43 smithi007 podman[98985]: 2024-09-17 00:13:43.33048048 +0000 UTC m=+6.048524200 container cleanup 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97) 2024-09-17T00:13:43.520 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:43 smithi007 bash[98985]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3 2024-09-17T00:13:43.849 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:43 smithi007 podman[99001]: 2024-09-17 00:13:43.523523139 +0000 UTC m=+0.558632165 container remove 6956484a2f3814e5380a6c45393877366c7efa6d7cce954e3b2eff545e7e69d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-17T00:13:43.901 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:43 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:43.506+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:44.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:43 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:43.903+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:44.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:43 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:43.952+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:44.207 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:43 smithi007 podman[99095]: 2024-09-17 00:13:43.860447424 +0000 UTC m=+0.016436358 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:13:44.207 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:43 smithi007 podman[99095]: 2024-09-17 00:13:43.982894356 +0000 UTC m=+0.138883289 container create 6db5af04533cff8d5a09c5f125c8f4ae4383addb9561dc998a51bfba849056f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=squid-c2ddf97, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:13:44.207 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:44 smithi007 podman[99095]: 2024-09-17 00:13:44.210219514 +0000 UTC m=+0.366208455 container init 6db5af04533cff8d5a09c5f125c8f4ae4383addb9561dc998a51bfba849056f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, RELEASE=squid-c2ddf97, GIT_CLEAN=True) 2024-09-17T00:13:44.541 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:44 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:44.237+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:44.599 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:44 smithi007 podman[99095]: 2024-09-17 00:13:44.213647962 +0000 UTC m=+0.369636898 container start 6db5af04533cff8d5a09c5f125c8f4ae4383addb9561dc998a51bfba849056f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-17T00:13:44.599 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:44 smithi007 podman[99095]: 2024-09-17 00:13:44.285178803 +0000 UTC m=+0.441167753 container attach 6db5af04533cff8d5a09c5f125c8f4ae4383addb9561dc998a51bfba849056f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-17T00:13:44.599 INFO:journalctl@ceph.osd.3.smithi007.stdout:Sep 17 00:13:44 smithi007 podman[99095]: 2024-09-17 00:13:44.405497058 +0000 UTC m=+0.561486188 container died 6db5af04533cff8d5a09c5f125c8f4ae4383addb9561dc998a51bfba849056f4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:13:44.908 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:44 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:44.542+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:44.969 DEBUG:teuthology.orchestra.run.smithi007:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.3.service' 2024-09-17T00:13:45.010 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:45.010 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-17T00:13:45.010 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-17T00:13:45.010 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.4 2024-09-17T00:13:45.232 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:45 smithi194 systemd[1]: Stopping Ceph osd.4 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:45.233 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:44 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:44.910+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:45.233 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:44 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:44.973+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:45.581 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:45.233+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:45.581 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:45.340+0000 7fdeddc0a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:45.581 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:45.340+0000 7fdeddc0a640 -1 osd.4 159 *** Got signal Terminated *** 2024-09-17T00:13:45.581 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:45.340+0000 7fdeddc0a640 -1 osd.4 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:13:45.908 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:45.582+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:46.249 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:45.950+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:46.250 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:45 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:45.974+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:46.546 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:46 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:46.251+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:46.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:46 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:46.548+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:47.270 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:46 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:46.916+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:47.270 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:47 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:47.015+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:47.554 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:47 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:47.272+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:47.908 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:47 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:47.555+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:48.249 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:47 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:47.955+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:48.249 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:48.023+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:48.535 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:48.251+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:48.908 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:48.537+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:48.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:48.537+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:22.075835+0000 front 2024-09-17T00:13:22.075879+0000 (oldest deadline 2024-09-17T00:13:47.975674+0000) 2024-09-17T00:13:49.211 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:48.990+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:49.211 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:48.944+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:49.211 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:48 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:48.944+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:49.513 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:49.213+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:49.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:49.515+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.375367+0000 front 2024-09-17T00:13:17.375530+0000 (oldest deadline 2024-09-17T00:13:42.075306+0000) 2024-09-17T00:13:49.909 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4[56370]: 2024-09-17T00:13:49.515+0000 7fded6218640 -1 osd.4 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:22.075835+0000 front 2024-09-17T00:13:22.075879+0000 (oldest deadline 2024-09-17T00:13:47.975674+0000) 2024-09-17T00:13:50.408 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:49.971+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:50.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:50 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:50.198+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:50.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:50 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:50.198+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:50.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:49.962+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:50.410 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:49 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:49.962+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:51.015 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:50 smithi194 podman[72628]: 2024-09-17 00:13:50.742220343 +0000 UTC m=+5.523665162 container died 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:13:51.016 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:50 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:50.969+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:51.016 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:50 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:50.969+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:51.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:51 smithi194 podman[72628]: 2024-09-17 00:13:51.208461757 +0000 UTC m=+5.989906569 container cleanup 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-17T00:13:51.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:51 smithi194 bash[72628]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4 2024-09-17T00:13:51.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:51.172+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:51.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:51.172+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:51.409 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:51.016+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:51.410 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:51.016+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:51.812 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:51 smithi194 podman[72676]: 2024-09-17 00:13:51.468609997 +0000 UTC m=+0.721181802 container remove 508bdb2e794c6a9f85b6df6e0656700fad12f290a9f0c6f29a9b6c12daab9226 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T00:13:52.073 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:51.963+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:52.073 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:51 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:51.963+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:52.073 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:52 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:52.022+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:52.073 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:52 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:52.022+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:52.074 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:51 smithi194 podman[72766]: 2024-09-17 00:13:51.718216538 +0000 UTC m=+0.016121754 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:13:52.074 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:51 smithi194 podman[72766]: 2024-09-17 00:13:51.844558037 +0000 UTC m=+0.142463239 container create 22631f699048854d4bfff92a85cbd350dfa72120cd4e6f8a1d98b1b41df58bd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-17T00:13:52.074 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:52 smithi194 podman[72766]: 2024-09-17 00:13:52.076022263 +0000 UTC m=+0.373927480 container init 22631f699048854d4bfff92a85cbd350dfa72120cd4e6f8a1d98b1b41df58bd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-c2ddf97, ceph=True, CEPH_POINT_RELEASE=) 2024-09-17T00:13:52.408 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:52 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:52.195+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:52.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:52 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:52.195+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:52.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:52 smithi194 podman[72766]: 2024-09-17 00:13:52.079690657 +0000 UTC m=+0.377595860 container start 22631f699048854d4bfff92a85cbd350dfa72120cd4e6f8a1d98b1b41df58bd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, RELEASE=squid-c2ddf97) 2024-09-17T00:13:52.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:52 smithi194 podman[72766]: 2024-09-17 00:13:52.163442128 +0000 UTC m=+0.461347338 container attach 22631f699048854d4bfff92a85cbd350dfa72120cd4e6f8a1d98b1b41df58bd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:13:52.409 INFO:journalctl@ceph.osd.4.smithi194.stdout:Sep 17 00:13:52 smithi194 podman[72766]: 2024-09-17 00:13:52.274833028 +0000 UTC m=+0.572738249 container died 22631f699048854d4bfff92a85cbd350dfa72120cd4e6f8a1d98b1b41df58bd4 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-17T00:13:52.860 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.4.service' 2024-09-17T00:13:52.899 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:13:52.899 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-17T00:13:52.900 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-17T00:13:52.900 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.5 2024-09-17T00:13:53.147 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:52 smithi194 systemd[1]: Stopping Ceph osd.5 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:13:53.147 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:53.007+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:53.147 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:53.007+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:53.148 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:52 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:52.943+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:53.148 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:52 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:52.943+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:53.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:53.148+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:53.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:53.148+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:53.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:53.254+0000 7ffbe2c85640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:13:53.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:53.254+0000 7ffbe2c85640 -1 osd.5 159 *** Got signal Terminated *** 2024-09-17T00:13:53.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:53.254+0000 7ffbe2c85640 -1 osd.5 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:13:54.182 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:53.972+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:54.182 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:53.972+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:54.183 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:53.923+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:54.183 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:53 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:53.923+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:54.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:54.184+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:54.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:54.184+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:55.218 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:54.950+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:55.244 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:54.950+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:55.244 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:54.975+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:55.244 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:54 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:54.975+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:55.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:55.220+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:55.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:55.220+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:56.192 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:55.930+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:56.192 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:55.930+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:56.192 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:55.914+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:56.192 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:55.914+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:56.192 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:55 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:55.914+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:13:56.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:56.194+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:56.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:56.194+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:57.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:56.923+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:57.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:56.923+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:57.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:56.876+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:57.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:56.876+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:57.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:56 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:56.876+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:13:57.658 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:57.205+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:57.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:57.205+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:57.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:57.205+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:32.005059+0000 front 2024-09-17T00:13:32.005046+0000 (oldest deadline 2024-09-17T00:13:56.704930+0000) 2024-09-17T00:13:58.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:57.893+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:58.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:57.893+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:58.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:57.844+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:58.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:57.844+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:58.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:57 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:57.844+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:13:58.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:58.235+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:16.902889+0000 front 2024-09-17T00:13:16.902783+0000 (oldest deadline 2024-09-17T00:13:40.402671+0000) 2024-09-17T00:13:58.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:58.235+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.703654+0000 front 2024-09-17T00:13:25.703510+0000 (oldest deadline 2024-09-17T00:13:49.803484+0000) 2024-09-17T00:13:58.659 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5[58623]: 2024-09-17T00:13:58.235+0000 7ffbdb293640 -1 osd.5 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:32.005059+0000 front 2024-09-17T00:13:32.005046+0000 (oldest deadline 2024-09-17T00:13:56.704930+0000) 2024-09-17T00:13:59.008 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:58 smithi194 podman[72880]: 2024-09-17 00:13:58.727147072 +0000 UTC m=+5.573259146 container died cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:13:59.008 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:58.826+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:13:59.008 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:58.826+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:13:59.008 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:58.826+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:13:59.009 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:58.935+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:13:59.009 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:58 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:58.935+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:13:59.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:59 smithi194 podman[72880]: 2024-09-17 00:13:59.196700719 +0000 UTC m=+6.042812796 container cleanup cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0) 2024-09-17T00:13:59.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:59 smithi194 bash[72880]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5 2024-09-17T00:13:59.797 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:59 smithi194 podman[72898]: 2024-09-17 00:13:59.457851283 +0000 UTC m=+0.729602466 container remove cd9a4795f158ae83557eff1ad785cfbe7b0510a17d879ea10e99b2e84c919937 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux ) 2024-09-17T00:14:00.049 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:59 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:59.939+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:00.049 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:59 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:59.939+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:00.049 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:13:59 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:13:59.939+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:00.050 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:59 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:59.800+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:00.050 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:59 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:59.800+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:00.050 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:13:59 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:13:59.800+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:00.050 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:59 smithi194 podman[72989]: 2024-09-17 00:13:59.721509233 +0000 UTC m=+0.016498232 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:14:00.050 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:13:59 smithi194 podman[72989]: 2024-09-17 00:13:59.841945182 +0000 UTC m=+0.136934166 container create c3af9d8ffd6b752402a436a01ef1c1214e1c7c6660116cbc8f84bf0fc19f4272 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-17T00:14:00.050 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:14:00 smithi194 podman[72989]: 2024-09-17 00:14:00.052695606 +0000 UTC m=+0.347684596 container init c3af9d8ffd6b752402a436a01ef1c1214e1c7c6660116cbc8f84bf0fc19f4272 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-17T00:14:00.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:14:00 smithi194 podman[72989]: 2024-09-17 00:14:00.055949074 +0000 UTC m=+0.350938059 container start c3af9d8ffd6b752402a436a01ef1c1214e1c7c6660116cbc8f84bf0fc19f4272 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=squid-c2ddf97, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-17T00:14:00.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:14:00 smithi194 podman[72989]: 2024-09-17 00:14:00.127544217 +0000 UTC m=+0.422533211 container attach c3af9d8ffd6b752402a436a01ef1c1214e1c7c6660116cbc8f84bf0fc19f4272 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:14:00.409 INFO:journalctl@ceph.osd.5.smithi194.stdout:Sep 17 00:14:00 smithi194 podman[72989]: 2024-09-17 00:14:00.245012179 +0000 UTC m=+0.540001180 container died c3af9d8ffd6b752402a436a01ef1c1214e1c7c6660116cbc8f84bf0fc19f4272 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-17T00:14:00.909 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:00 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:00.810+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:00.909 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:00 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:00.810+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:00.909 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:00 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:00.810+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:00.975 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.5.service' 2024-09-17T00:14:01.019 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:14:01.019 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-17T00:14:01.019 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-17T00:14:01.019 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.6 2024-09-17T00:14:01.351 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 systemd[1]: Stopping Ceph osd.6 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:14:01.351 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:00 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:00.928+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:01.351 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:00 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:00.928+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:01.351 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:00 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:00.928+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:01.658 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:01.353+0000 7f025560f640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:14:01.658 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:01.353+0000 7f025560f640 -1 osd.6 159 *** Got signal Terminated *** 2024-09-17T00:14:01.658 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:01.353+0000 7f025560f640 -1 osd.6 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:14:02.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:01.810+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:02.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:01.810+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:02.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:01.810+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:02.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:01.924+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:02.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:01.924+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:02.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:01 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:01.924+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:03.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:02 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:02.844+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:03.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:02 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:02.844+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:03.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:02 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:02.844+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:03.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:02 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:02.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:03.160 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:02 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:02.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:03.160 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:02 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:02.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:04.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:03.796+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:04.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:03.796+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:04.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:03.796+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:04.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:03.905+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:04.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:03.905+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:04.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:03 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:03.905+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:04.796+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:04.796+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:04.796+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:04.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:04.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:04.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:05.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:04 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:04.891+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:06.158 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:05.779+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:17.961696+0000 front 2024-09-17T00:13:17.961670+0000 (oldest deadline 2024-09-17T00:13:39.061447+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:05.779+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:24.362569+0000 front 2024-09-17T00:13:24.362555+0000 (oldest deadline 2024-09-17T00:13:48.462375+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:05.779+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:34.263728+0000 front 2024-09-17T00:13:34.263777+0000 (oldest deadline 2024-09-17T00:13:55.363668+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6[60891]: 2024-09-17T00:14:05.779+0000 7f0251c25640 -1 osd.6 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:40.664700+0000 front 2024-09-17T00:13:40.664650+0000 (oldest deadline 2024-09-17T00:14:05.364424+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:05.878+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:05.878+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:05.878+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:06.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:05 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:05.878+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:07.073 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:06 smithi194 podman[73090]: 2024-09-17 00:14:06.812860996 +0000 UTC m=+5.561976556 container died 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:14:07.073 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:06 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:06.866+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:07.073 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:06 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:06.866+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:07.073 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:06 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:06.866+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:07.073 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:06 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:06.866+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:07.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:07 smithi194 podman[73090]: 2024-09-17 00:14:07.227032789 +0000 UTC m=+5.976148337 container cleanup 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-17T00:14:07.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:07 smithi194 bash[73090]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6 2024-09-17T00:14:07.857 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:07 smithi194 podman[73105]: 2024-09-17 00:14:07.446926673 +0000 UTC m=+0.635881338 container remove 81dd179cd96227d59bbed85248f3496f0c589deea7457e231ee956a8d35865fe (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-17T00:14:08.131 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:07 smithi194 podman[73224]: 2024-09-17 00:14:07.788399439 +0000 UTC m=+0.016401766 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:14:08.132 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:07 smithi194 podman[73224]: 2024-09-17 00:14:07.926495159 +0000 UTC m=+0.154497486 container create fd3597200c19afaf6cdef10fe778bf31d3f17dac08803c1e5274199be7174354 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, GIT_CLEAN=True, RELEASE=squid-c2ddf97, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-17T00:14:08.132 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:08 smithi194 podman[73224]: 2024-09-17 00:14:08.134620145 +0000 UTC m=+0.362622477 container init fd3597200c19afaf6cdef10fe778bf31d3f17dac08803c1e5274199be7174354 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:14:08.132 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:07 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:07.860+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:08.132 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:07 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:07.860+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:08.132 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:07 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:07.860+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:08.132 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:07 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:07.860+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:08.408 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:08 smithi194 podman[73224]: 2024-09-17 00:14:08.13786733 +0000 UTC m=+0.365869660 container start fd3597200c19afaf6cdef10fe778bf31d3f17dac08803c1e5274199be7174354 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-17T00:14:08.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:08 smithi194 podman[73224]: 2024-09-17 00:14:08.217870523 +0000 UTC m=+0.445872851 container attach fd3597200c19afaf6cdef10fe778bf31d3f17dac08803c1e5274199be7174354 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-17T00:14:08.409 INFO:journalctl@ceph.osd.6.smithi194.stdout:Sep 17 00:14:08 smithi194 podman[73224]: 2024-09-17 00:14:08.331067497 +0000 UTC m=+0.559069824 container died fd3597200c19afaf6cdef10fe778bf31d3f17dac08803c1e5274199be7174354 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-17T00:14:08.883 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.6.service' 2024-09-17T00:14:08.921 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:14:08.921 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-17T00:14:08.921 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-17T00:14:08.921 DEBUG:teuthology.orchestra.run.smithi194:> sudo systemctl stop ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.7 2024-09-17T00:14:09.158 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:08 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:08.900+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:09.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:08 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:08.900+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:09.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:08 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:08.900+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:09.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:08 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:08.900+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:09.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 systemd[1]: Stopping Ceph osd.7 for 5c15e83c-7486-11ef-bceb-c7b262605968... 2024-09-17T00:14:09.658 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.265+0000 7fa0feef8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-17T00:14:09.658 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.265+0000 7fa0feef8640 -1 osd.7 159 *** Got signal Terminated *** 2024-09-17T00:14:09.659 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.265+0000 7fa0feef8640 -1 osd.7 159 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-17T00:14:10.158 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.853+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:10.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.853+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:10.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.853+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:10.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:09 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:09.853+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:10.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:10 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:10.812+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:10.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:10 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:10.813+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:10.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:10 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:10.813+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:10.909 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:10 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:10.813+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:12.158 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:11.847+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:12.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:11.847+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:12.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:11.847+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:12.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:11 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:11.847+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:13.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:12 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:12.831+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:13.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:12 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:12.831+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:13.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:12 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:12.831+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:13.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:12 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:12.831+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:14.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:13 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:13.821+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6806 osd.0 since back 2024-09-17T00:13:14.862943+0000 front 2024-09-17T00:13:14.862871+0000 (oldest deadline 2024-09-17T00:13:40.162764+0000) 2024-09-17T00:14:14.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:13 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:13.821+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6814 osd.1 since back 2024-09-17T00:13:25.364370+0000 front 2024-09-17T00:13:25.364325+0000 (oldest deadline 2024-09-17T00:13:50.064152+0000) 2024-09-17T00:14:14.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:13 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:13.821+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6822 osd.2 since back 2024-09-17T00:13:33.465731+0000 front 2024-09-17T00:13:33.465681+0000 (oldest deadline 2024-09-17T00:13:59.365632+0000) 2024-09-17T00:14:14.159 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:13 smithi194 ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7[63163]: 2024-09-17T00:14:13.821+0000 7fa0fb50e640 -1 osd.7 159 heartbeat_check: no reply from 172.21.15.7:6830 osd.3 since back 2024-09-17T00:13:39.866824+0000 front 2024-09-17T00:13:39.866702+0000 (oldest deadline 2024-09-17T00:14:04.566728+0000) 2024-09-17T00:14:14.964 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:14 smithi194 podman[73326]: 2024-09-17 00:14:14.694465269 +0000 UTC m=+5.551644690 container died eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, RELEASE=squid-c2ddf97, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-17T00:14:15.329 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73326]: 2024-09-17 00:14:15.065411925 +0000 UTC m=+5.922591346 container cleanup eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=squid-c2ddf97, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-17T00:14:15.329 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 bash[73326]: ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7 2024-09-17T00:14:15.658 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73356]: 2024-09-17 00:14:15.332186355 +0000 UTC m=+0.631490660 container remove eac97979e96ce4e5265eb28a7f66fc40dbfc7d87086cd601aa340235e9f69075 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=squid-c2ddf97, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-17T00:14:15.965 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73448]: 2024-09-17 00:14:15.683038356 +0000 UTC m=+0.107015239 container create cb20e12a9f0b0d0b602a9652657d5439b69c25b13fff5d75466c3ec8cbf0f7a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-17T00:14:15.966 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73448]: 2024-09-17 00:14:15.591605774 +0000 UTC m=+0.015582625 image pull ebb0615149def4a3179c82c0cc47af75808300abb9f1142788e5e605a7ac0e5b quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3 2024-09-17T00:14:15.966 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73448]: 2024-09-17 00:14:15.893589679 +0000 UTC m=+0.317566519 container init cb20e12a9f0b0d0b602a9652657d5439b69c25b13fff5d75466c3ec8cbf0f7a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-17T00:14:15.966 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73448]: 2024-09-17 00:14:15.897013222 +0000 UTC m=+0.320990071 container start cb20e12a9f0b0d0b602a9652657d5439b69c25b13fff5d75466c3ec8cbf0f7a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=squid-c2ddf97) 2024-09-17T00:14:15.966 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:15 smithi194 podman[73448]: 2024-09-17 00:14:15.968417986 +0000 UTC m=+0.392394826 container attach cb20e12a9f0b0d0b602a9652657d5439b69c25b13fff5d75466c3ec8cbf0f7a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True) 2024-09-17T00:14:16.402 INFO:journalctl@ceph.osd.7.smithi194.stdout:Sep 17 00:14:16 smithi194 podman[73448]: 2024-09-17 00:14:16.096950513 +0000 UTC m=+0.520927354 container died cb20e12a9f0b0d0b602a9652657d5439b69c25b13fff5d75466c3ec8cbf0f7a9 (image=quay.ceph.io/ceph-ci/ceph@sha256:475e68c989240ebfb6ca26a687a713828159881ccf6edcf69b7320f1853af3e3, name=ceph-5c15e83c-7486-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=squid-c2ddf97, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-17T00:14:16.701 DEBUG:teuthology.orchestra.run.smithi194:> sudo pkill -f 'journalctl -f -n 0 -u ceph-5c15e83c-7486-11ef-bceb-c7b262605968@osd.7.service' 2024-09-17T00:14:16.745 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-17T00:14:16.746 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-17T00:14:16.746 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T00:17:15.617 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-17T00:17:37.005 DEBUG:teuthology.orchestra.run.smithi007:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:17:37.041 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-17T00:17:37.068 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-17T00:17:37.070 DEBUG:teuthology.misc:Transferring archived files from smithi007:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895/remote/smithi007/crash 2024-09-17T00:17:37.071 DEBUG:teuthology.orchestra.run.smithi007:> sudo tar c -f - -C /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/crash -- . 2024-09-17T00:17:37.126 INFO:teuthology.orchestra.run.smithi007.stderr:tar: /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T00:17:37.126 INFO:teuthology.orchestra.run.smithi007.stderr:tar: Error is not recoverable: exiting now 2024-09-17T00:17:37.128 DEBUG:teuthology.misc:Transferring archived files from smithi194:/var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895/remote/smithi194/crash 2024-09-17T00:17:37.129 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/crash -- . 2024-09-17T00:17:37.165 INFO:teuthology.orchestra.run.smithi194.stderr:tar: /var/lib/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-17T00:17:37.165 INFO:teuthology.orchestra.run.smithi194.stderr:tar: Error is not recoverable: exiting now 2024-09-17T00:17:37.166 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-17T00:17:37.166 DEBUG:teuthology.orchestra.run.smithi007:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-17T00:17:37.200 INFO:tasks.cephadm:Compressing logs... 2024-09-17T00:17:37.200 DEBUG:teuthology.orchestra.run.smithi007:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T00:17:37.243 DEBUG:teuthology.orchestra.run.smithi194:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-17T00:17:37.270 INFO:teuthology.orchestra.run.smithi194.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mgr.smithi194.xlteft.log 2024-09-17T00:17:37.270 INFO:teuthology.orchestra.run.smithi194.stderr:: No such file or directory 2024-09-17T00:17:37.270 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-17T00:17:37.270 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mgr.smithi194.xlteft.log: /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mon.smithi194.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mon.smithi194.log: /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi194.emredf.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi194.emredf.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-17T00:17:37.271 INFO:teuthology.orchestra.run.smithi194.stderr: 87.9% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi194.emredf.log.gz 2024-09-17T00:17:37.277 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.cephadm.log: 85.5% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T00:17:37.277 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-17T00:17:37.277 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log 2024-09-17T00:17:37.277 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T00:17:37.277 INFO:teuthology.orchestra.run.smithi194.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T00:17:37.279 INFO:teuthology.orchestra.run.smithi007.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-17T00:17:37.279 INFO:teuthology.orchestra.run.smithi007.stderr:: No such file or directory 2024-09-17T00:17:37.279 INFO:teuthology.orchestra.run.smithi007.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mon.smithi007.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mon.smithi007.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mgr.smithi007.czdphd.log 2024-09-17T00:17:37.280 INFO:teuthology.orchestra.run.smithi007.stderr: 85.6% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-17T00:17:37.281 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi007.fkkfbl.log 2024-09-17T00:17:37.281 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mgr.smithi007.czdphd.log: 91.5% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-17T00:17:37.281 INFO:teuthology.orchestra.run.smithi007.stderr:gzip -5 --verbose -- /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-17T00:17:37.281 INFO:teuthology.orchestra.run.smithi007.stderr: 87.0% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T00:17:37.281 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi007.fkkfbl.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-17T00:17:37.282 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log: 87.0% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-17T00:17:37.282 INFO:teuthology.orchestra.run.smithi194.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T00:17:37.285 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.3.log: 91.1% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-17T00:17:37.285 INFO:teuthology.orchestra.run.smithi194.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T00:17:37.302 INFO:teuthology.orchestra.run.smithi007.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi007.fkkfbl.log.gz 2024-09-17T00:17:37.324 INFO:teuthology.orchestra.run.smithi007.stderr: 91.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-17T00:17:37.336 INFO:teuthology.orchestra.run.smithi194.stderr: 89.8% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mgr.smithi194.xlteft.log.gz 2024-09-17T00:17:37.536 INFO:teuthology.orchestra.run.smithi007.stderr: 88.7% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mgr.smithi007.czdphd.log.gz 2024-09-17T00:17:37.707 INFO:teuthology.orchestra.run.smithi194.stderr: 92.6% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mon.smithi194.log.gz 2024-09-17T00:17:38.610 INFO:teuthology.orchestra.run.smithi007.stderr: 91.8% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-mon.smithi007.log.gz 2024-09-17T00:17:49.539 INFO:teuthology.orchestra.run.smithi194.stderr: 93.9% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-17T00:17:50.237 INFO:teuthology.orchestra.run.smithi007.stderr: 94.5% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-17T00:17:51.031 INFO:teuthology.orchestra.run.smithi194.stderr: 94.2% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-17T00:17:52.770 INFO:teuthology.orchestra.run.smithi007.stderr: 94.1% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-17T00:17:54.107 INFO:teuthology.orchestra.run.smithi007.stderr: 94.1% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-17T00:17:54.177 INFO:teuthology.orchestra.run.smithi007.stderr: 94.2% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-17T00:17:54.179 INFO:teuthology.orchestra.run.smithi007.stderr: 2024-09-17T00:17:54.179 INFO:teuthology.orchestra.run.smithi007.stderr:real 0m16.923s 2024-09-17T00:17:54.180 INFO:teuthology.orchestra.run.smithi007.stderr:user 1m1.565s 2024-09-17T00:17:54.180 INFO:teuthology.orchestra.run.smithi007.stderr:sys 0m2.242s 2024-09-17T00:17:55.154 INFO:teuthology.orchestra.run.smithi194.stderr: 94.2% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-17T00:17:57.112 INFO:teuthology.orchestra.run.smithi194.stderr: 94.1% -- replaced with /var/log/ceph/5c15e83c-7486-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-17T00:17:57.114 INFO:teuthology.orchestra.run.smithi194.stderr: 2024-09-17T00:17:57.114 INFO:teuthology.orchestra.run.smithi194.stderr:real 0m19.856s 2024-09-17T00:17:57.114 INFO:teuthology.orchestra.run.smithi194.stderr:user 1m1.708s 2024-09-17T00:17:57.114 INFO:teuthology.orchestra.run.smithi194.stderr:sys 0m2.472s 2024-09-17T00:17:57.114 INFO:tasks.cephadm:Archiving logs... 2024-09-17T00:17:57.115 DEBUG:teuthology.misc:Transferring archived files from smithi007:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895/remote/smithi007/log 2024-09-17T00:17:57.116 DEBUG:teuthology.orchestra.run.smithi007:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T00:18:05.410 DEBUG:teuthology.misc:Transferring archived files from smithi194:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895/remote/smithi194/log 2024-09-17T00:18:05.411 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-17T00:18:14.612 INFO:tasks.cephadm:Removing cluster... 2024-09-17T00:18:14.612 DEBUG:teuthology.orchestra.run.smithi007:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 --force 2024-09-17T00:18:14.983 DEBUG:teuthology.orchestra.run.smithi194:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 5c15e83c-7486-11ef-bceb-c7b262605968 --force 2024-09-17T00:18:15.338 INFO:tasks.cephadm:Removing cephadm ... 2024-09-17T00:18:15.338 DEBUG:teuthology.orchestra.run.smithi007:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T00:18:15.355 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-17T00:18:15.370 INFO:tasks.cephadm:Teardown complete 2024-09-17T00:18:15.371 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-17T00:18:15.381 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-17T00:18:15.381 DEBUG:teuthology.orchestra.run.smithi007:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T00:18:15.397 DEBUG:teuthology.orchestra.run.smithi194:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-17T00:18:15.411 INFO:teuthology.orchestra.run.smithi007.stderr:bash: line 1: ntpq: command not found 2024-09-17T00:18:15.416 INFO:teuthology.orchestra.run.smithi007.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T00:18:15.416 INFO:teuthology.orchestra.run.smithi007.stdout:=============================================================================== 2024-09-17T00:18:15.416 INFO:teuthology.orchestra.run.smithi007.stdout:^* hv01.front.sepia.ceph.com 3 6 377 16 +4276us[+4314us] +/- 67ms 2024-09-17T00:18:15.416 INFO:teuthology.orchestra.run.smithi007.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 17 -3170us[-3132us] +/- 66ms 2024-09-17T00:18:15.416 INFO:teuthology.orchestra.run.smithi007.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 16 -1447us[-1447us] +/- 90ms 2024-09-17T00:18:15.416 INFO:teuthology.orchestra.run.smithi007.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:18:15.425 INFO:teuthology.orchestra.run.smithi194.stderr:bash: line 1: ntpq: command not found 2024-09-17T00:18:15.428 INFO:teuthology.orchestra.run.smithi194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-17T00:18:15.429 INFO:teuthology.orchestra.run.smithi194.stdout:=============================================================================== 2024-09-17T00:18:15.429 INFO:teuthology.orchestra.run.smithi194.stdout:^* hv01.front.sepia.ceph.com 3 6 377 18 +4234us[+4279us] +/- 67ms 2024-09-17T00:18:15.429 INFO:teuthology.orchestra.run.smithi194.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 17 -3190us[-3190us] +/- 66ms 2024-09-17T00:18:15.429 INFO:teuthology.orchestra.run.smithi194.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 19 -1493us[-1447us] +/- 90ms 2024-09-17T00:18:15.429 INFO:teuthology.orchestra.run.smithi194.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-17T00:18:15.430 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-17T00:18:15.439 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-17T00:18:15.440 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-17T00:18:15.450 DEBUG:teuthology.orchestra.run.smithi007:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T00:18:15.539 DEBUG:teuthology.orchestra.run.smithi194:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-17T00:18:15.612 DEBUG:teuthology.orchestra.run.smithi007:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-17T00:18:15.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:18:15.642 DEBUG:teuthology.orchestra.run.smithi194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-09-17T00:18:15.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:18:15.670 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-17T00:18:15.681 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-17T00:18:15.690 INFO:teuthology.task.internal:Duration was 1932.297072 seconds 2024-09-17T00:18:15.691 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-17T00:18:15.700 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-17T00:18:15.700 DEBUG:teuthology.orchestra.run.smithi007:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T00:18:15.703 DEBUG:teuthology.orchestra.run.smithi194:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-17T00:18:15.741 INFO:teuthology.orchestra.run.smithi007.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T00:18:15.748 INFO:teuthology.orchestra.run.smithi194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-17T00:18:16.193 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-17T00:18:16.193 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi007.front.sepia.ceph.com 2024-09-17T00:18:16.193 DEBUG:teuthology.orchestra.run.smithi007:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T00:18:16.216 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi194.front.sepia.ceph.com 2024-09-17T00:18:16.217 DEBUG:teuthology.orchestra.run.smithi194:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-17T00:18:16.253 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-17T00:18:16.253 DEBUG:teuthology.orchestra.run.smithi007:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T00:18:16.259 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-17T00:18:16.320 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-17T00:18:16.320 DEBUG:teuthology.orchestra.run.smithi007:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T00:18:16.328 DEBUG:teuthology.orchestra.run.smithi194:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-17T00:18:31.301 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-17T00:18:31.314 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-17T00:18:31.314 DEBUG:teuthology.orchestra.run.smithi007:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T00:18:31.339 DEBUG:teuthology.orchestra.run.smithi194:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-17T00:18:31.370 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-17T00:18:31.380 DEBUG:teuthology.orchestra.run.smithi007:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T00:18:31.383 DEBUG:teuthology.orchestra.run.smithi194:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-17T00:18:31.404 INFO:teuthology.orchestra.run.smithi007.stdout:kernel.core_pattern = core 2024-09-17T00:18:31.438 INFO:teuthology.orchestra.run.smithi194.stdout:kernel.core_pattern = core 2024-09-17T00:18:31.462 DEBUG:teuthology.orchestra.run.smithi007:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T00:18:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:18:31.485 DEBUG:teuthology.orchestra.run.smithi194:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-17T00:18:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-17T00:18:31.519 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-17T00:18:31.529 INFO:teuthology.task.internal:Transferring archived files... 2024-09-17T00:18:31.531 DEBUG:teuthology.misc:Transferring archived files from smithi007:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895/remote/smithi007 2024-09-17T00:18:31.531 DEBUG:teuthology.orchestra.run.smithi007:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T00:18:31.605 DEBUG:teuthology.misc:Transferring archived files from smithi194:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-16_21:08:04-orch-squid-distro-default-smithi/7907895/remote/smithi194 2024-09-17T00:18:31.606 DEBUG:teuthology.orchestra.run.smithi194:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-17T00:18:31.647 INFO:teuthology.task.internal:Removing archive directory... 2024-09-17T00:18:31.647 DEBUG:teuthology.orchestra.run.smithi007:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T00:18:31.650 DEBUG:teuthology.orchestra.run.smithi194:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-17T00:18:31.697 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-17T00:18:31.710 INFO:teuthology.task.internal:Not uploading archives. 2024-09-17T00:18:31.710 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-17T00:18:31.721 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-17T00:18:31.721 DEBUG:teuthology.orchestra.run.smithi007:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T00:18:31.724 DEBUG:teuthology.orchestra.run.smithi194:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-17T00:18:31.737 INFO:teuthology.orchestra.run.smithi007.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 00:18 /home/ubuntu/cephtest 2024-09-17T00:18:31.753 INFO:teuthology.orchestra.run.smithi194.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 17 00:18 /home/ubuntu/cephtest 2024-09-17T00:18:31.754 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-17T00:18:31.766 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-17T00:18:31.794 INFO:teuthology.run:Summary data: description: orch/cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 1932.2970724105835 owner: scheduled_teuthology@teuthology success: true 2024-09-17T00:18:31.795 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-17T00:18:31.875 INFO:teuthology.run:pass