2024-09-13T12:36:46.181 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T12:36:46.202 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T12:36:46.294 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574 branch: wip-guits-main-2024-09-13-0840 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: gabrioux@ibm.com first_in_suite: false job_id: '7903574' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-guits-main-2024-09-13-0840 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - CEPHADM_REFRESH_FAILED log-only-match: - CEPHADM_ sha1: b0201b8c79733293453e7f10a10c7fa43119222b ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: b0201b8c79733293453e7f10a10c7fa43119222b selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: wip-guits-main-2024-09-13-0840 sha1: b0201b8c79733293453e7f10a10c7fa43119222b owner: scheduled_gabrioux@teuthology priority: 70 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 4031 sha1: b0201b8c79733293453e7f10a10c7fa43119222b sleep_before_teardown: 0 subset: 1/8 suite: orch:cephadm suite_branch: wip-guits-main-2024-09-13-0840 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_b0201b8c79733293453e7f10a10c7fa43119222b/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: b0201b8c79733293453e7f10a10c7fa43119222b targets: smithi042.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAxtwUlwn67f2xW5ancmaz+zfePCji45uN30irAEVkvqYTjtnLexE1OH2L3+/J78sVEVeJpfgtdNEc7bIsogO7c= smithi046.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBGQya4lwm2IBssdqHlfo5KYZZ9LLG9IijTHmk+kiJgG2rLrnVQdZpta661uvcceUptCyqD2jPoR4orKrBD2k+M= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-13_12:30:34 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: gabrioux verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-09-13T12:36:46.294 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_b0201b8c79733293453e7f10a10c7fa43119222b/qa; will attempt to use it 2024-09-13T12:36:46.295 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_b0201b8c79733293453e7f10a10c7fa43119222b/qa/tasks 2024-09-13T12:36:46.295 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T12:36:46.297 INFO:teuthology.task.internal:Checking packages... 2024-09-13T12:36:46.320 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'b0201b8c79733293453e7f10a10c7fa43119222b' 2024-09-13T12:36:46.320 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T12:36:46.320 INFO:teuthology.packaging:ref: None 2024-09-13T12:36:46.320 INFO:teuthology.packaging:tag: None 2024-09-13T12:36:46.320 INFO:teuthology.packaging:branch: wip-guits-main-2024-09-13-0840 2024-09-13T12:36:46.320 INFO:teuthology.packaging:sha1: b0201b8c79733293453e7f10a10c7fa43119222b 2024-09-13T12:36:46.320 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-guits-main-2024-09-13-0840 2024-09-13T12:36:46.467 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4917.gb0201b8c 2024-09-13T12:36:46.469 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T12:36:46.481 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T12:36:46.481 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T12:36:46.499 INFO:teuthology.task.internal:Saving configuration 2024-09-13T12:36:46.513 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T12:36:46.525 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T12:36:46.558 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi042.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 12:31:45.768155', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAxtwUlwn67f2xW5ancmaz+zfePCji45uN30irAEVkvqYTjtnLexE1OH2L3+/J78sVEVeJpfgtdNEc7bIsogO7c='} 2024-09-13T12:36:46.586 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi046.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-13 12:31:45.766534', 'locked_by': 'scheduled_gabrioux@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBBGQya4lwm2IBssdqHlfo5KYZZ9LLG9IijTHmk+kiJgG2rLrnVQdZpta661uvcceUptCyqD2jPoR4orKrBD2k+M='} 2024-09-13T12:36:46.586 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T12:36:46.592 INFO:teuthology.task.internal:roles: ubuntu@smithi042.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-09-13T12:36:46.592 INFO:teuthology.task.internal:roles: ubuntu@smithi046.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-09-13T12:36:46.592 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T12:36:46.693 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f15ba117be0>, signals=[15]) 2024-09-13T12:36:46.693 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T12:36:46.698 INFO:teuthology.task.internal:Opening connections... 2024-09-13T12:36:46.699 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:36:46.700 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:36:46.788 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi046.front.sepia.ceph.com 2024-09-13T12:36:46.789 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:36:46.875 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T12:36:46.880 DEBUG:teuthology.orchestra.run.smithi042:> uname -m 2024-09-13T12:36:46.900 INFO:teuthology.orchestra.run.smithi042.stdout:x86_64 2024-09-13T12:36:46.900 DEBUG:teuthology.orchestra.run.smithi042:> cat /etc/os-release 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:NAME="CentOS Stream" 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:VERSION="9" 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:ID="centos" 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:ID_LIKE="rhel fedora" 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:VERSION_ID="9" 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:PLATFORM_ID="platform:el9" 2024-09-13T12:36:46.958 INFO:teuthology.orchestra.run.smithi042.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:ANSI_COLOR="0;31" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:LOGO="fedora-logo-icon" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:HOME_URL="https://centos.org/" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T12:36:46.959 INFO:teuthology.orchestra.run.smithi042.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T12:36:46.960 INFO:teuthology.lock.ops:Updating smithi042.front.sepia.ceph.com on lock server 2024-09-13T12:36:47.001 DEBUG:teuthology.orchestra.run.smithi046:> uname -m 2024-09-13T12:36:47.021 INFO:teuthology.orchestra.run.smithi046.stdout:x86_64 2024-09-13T12:36:47.021 DEBUG:teuthology.orchestra.run.smithi046:> cat /etc/os-release 2024-09-13T12:36:47.078 INFO:teuthology.orchestra.run.smithi046.stdout:NAME="CentOS Stream" 2024-09-13T12:36:47.078 INFO:teuthology.orchestra.run.smithi046.stdout:VERSION="9" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:ID="centos" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:ID_LIKE="rhel fedora" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:VERSION_ID="9" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:PLATFORM_ID="platform:el9" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:ANSI_COLOR="0;31" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:LOGO="fedora-logo-icon" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:HOME_URL="https://centos.org/" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-13T12:36:47.079 INFO:teuthology.orchestra.run.smithi046.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-13T12:36:47.080 INFO:teuthology.orchestra.run.smithi046.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-13T12:36:47.080 INFO:teuthology.lock.ops:Updating smithi046.front.sepia.ceph.com on lock server 2024-09-13T12:36:47.118 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T12:36:47.131 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T12:36:47.183 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T12:36:47.183 DEBUG:teuthology.orchestra.run.smithi042:> test '!' -e /home/ubuntu/cephtest 2024-09-13T12:36:47.186 DEBUG:teuthology.orchestra.run.smithi046:> test '!' -e /home/ubuntu/cephtest 2024-09-13T12:36:47.203 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T12:36:47.208 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T12:36:47.208 DEBUG:teuthology.orchestra.run.smithi042:> test -z $(ls -A /var/lib/ceph) 2024-09-13T12:36:47.243 DEBUG:teuthology.orchestra.run.smithi046:> test -z $(ls -A /var/lib/ceph) 2024-09-13T12:36:47.272 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T12:36:47.335 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T12:36:47.348 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T12:36:47.348 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T12:36:47.349 DEBUG:teuthology.orchestra.run.smithi042:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T12:36:47.349 DEBUG:teuthology.orchestra.run.smithi046:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T12:36:47.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T12:36:47.366 DEBUG:teuthology.orchestra.run.smithi046:> uname -r 2024-09-13T12:36:47.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T12:36:47.367 DEBUG:teuthology.orchestra.run.smithi042:> uname -r 2024-09-13T12:36:47.423 INFO:teuthology.orchestra.run.smithi046.stdout:5.14.0-503.el9.x86_64 2024-09-13T12:36:47.423 INFO:teuthology.task.kernel:Running kernel on smithi046: 5.14.0-503.el9.x86_64 2024-09-13T12:36:47.423 DEBUG:teuthology.orchestra.run.smithi046:> sudo yum install -y kernel 2024-09-13T12:36:47.425 INFO:teuthology.orchestra.run.smithi042.stdout:5.14.0-503.el9.x86_64 2024-09-13T12:36:47.425 INFO:teuthology.task.kernel:Running kernel on smithi042: 5.14.0-503.el9.x86_64 2024-09-13T12:36:47.425 DEBUG:teuthology.orchestra.run.smithi042:> sudo yum install -y kernel 2024-09-13T12:36:50.891 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.2 MB 00:01 2024-09-13T12:36:51.074 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - BaseOS 4.2 MB/s | 8.2 MB 00:01 2024-09-13T12:36:55.241 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - AppStream 6.8 MB/s | 20 MB 00:03 2024-09-13T12:36:59.658 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - AppStream 2.7 MB/s | 20 MB 00:07 2024-09-13T12:37:01.700 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - CRB 2.6 MB/s | 6.5 MB 00:02 2024-09-13T12:37:03.625 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - Extras packages 28 kB/s | 19 kB 00:00 2024-09-13T12:37:04.025 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - CRB 15 MB/s | 6.5 MB 00:00 2024-09-13T12:37:04.771 INFO:teuthology.orchestra.run.smithi046.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-13T12:37:05.910 INFO:teuthology.orchestra.run.smithi042.stdout:CentOS Stream 9 - Extras packages 30 kB/s | 19 kB 00:00 2024-09-13T12:37:06.764 INFO:teuthology.orchestra.run.smithi042.stdout:Extra Packages for Enterprise Linux 31 MB/s | 23 MB 00:00 2024-09-13T12:37:11.894 INFO:teuthology.orchestra.run.smithi046.stdout:lab-extras 1.1 kB/s | 1.7 kB 00:01 2024-09-13T12:37:12.428 INFO:teuthology.orchestra.run.smithi042.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-09-13T12:37:13.991 INFO:teuthology.orchestra.run.smithi046.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T12:37:13.991 INFO:teuthology.orchestra.run.smithi046.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T12:37:14.084 INFO:teuthology.orchestra.run.smithi046.stdout:Dependencies resolved. 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout: Package Architecture Version Repository Size 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout:Installing: 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout:Installing dependencies: 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:37:14.091 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction Summary 2024-09-13T12:37:14.092 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-09-13T12:37:14.092 INFO:teuthology.orchestra.run.smithi046.stdout:Install 4 Packages 2024-09-13T12:37:14.092 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:37:14.092 INFO:teuthology.orchestra.run.smithi046.stdout:Total download size: 79 M 2024-09-13T12:37:14.092 INFO:teuthology.orchestra.run.smithi046.stdout:Installed size: 126 M 2024-09-13T12:37:14.092 INFO:teuthology.orchestra.run.smithi046.stdout:Downloading Packages: 2024-09-13T12:37:14.403 INFO:teuthology.orchestra.run.smithi046.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 270 kB/s | 23 kB 00:00 2024-09-13T12:37:14.526 INFO:teuthology.orchestra.run.smithi042.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-13T12:37:14.526 INFO:teuthology.orchestra.run.smithi042.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-13T12:37:14.621 INFO:teuthology.orchestra.run.smithi042.stdout:Dependencies resolved. 2024-09-13T12:37:14.632 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T12:37:14.632 INFO:teuthology.orchestra.run.smithi042.stdout: Package Architecture Version Repository Size 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout:Installing: 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout:Installing dependencies: 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction Summary 2024-09-13T12:37:14.633 INFO:teuthology.orchestra.run.smithi042.stdout:================================================================================ 2024-09-13T12:37:14.634 INFO:teuthology.orchestra.run.smithi042.stdout:Install 4 Packages 2024-09-13T12:37:14.634 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:37:14.634 INFO:teuthology.orchestra.run.smithi042.stdout:Total download size: 79 M 2024-09-13T12:37:14.635 INFO:teuthology.orchestra.run.smithi042.stdout:Installed size: 126 M 2024-09-13T12:37:14.635 INFO:teuthology.orchestra.run.smithi042.stdout:Downloading Packages: 2024-09-13T12:37:14.787 INFO:teuthology.orchestra.run.smithi046.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-13T12:37:14.938 INFO:teuthology.orchestra.run.smithi042.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 191 kB/s | 23 kB 00:00 2024-09-13T12:37:15.179 INFO:teuthology.orchestra.run.smithi046.stdout:(3/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 40 MB/s | 35 MB 00:00 2024-09-13T12:37:15.238 INFO:teuthology.orchestra.run.smithi042.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 37 MB/s | 16 MB 00:00 2024-09-13T12:37:15.587 INFO:teuthology.orchestra.run.smithi046.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-13T12:37:15.587 INFO:teuthology.orchestra.run.smithi046.stdout:-------------------------------------------------------------------------------- 2024-09-13T12:37:15.587 INFO:teuthology.orchestra.run.smithi046.stdout:Total 53 MB/s | 79 MB 00:01 2024-09-13T12:37:15.730 INFO:teuthology.orchestra.run.smithi042.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 36 MB/s | 29 MB 00:00 2024-09-13T12:37:16.298 INFO:teuthology.orchestra.run.smithi046.stdout:Running transaction check 2024-09-13T12:37:16.305 INFO:teuthology.orchestra.run.smithi042.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 23 MB/s | 35 MB 00:01 2024-09-13T12:37:16.306 INFO:teuthology.orchestra.run.smithi042.stdout:-------------------------------------------------------------------------------- 2024-09-13T12:37:16.306 INFO:teuthology.orchestra.run.smithi042.stdout:Total 47 MB/s | 79 MB 00:01 2024-09-13T12:37:16.663 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction check succeeded. 2024-09-13T12:37:16.663 INFO:teuthology.orchestra.run.smithi046.stdout:Running transaction test 2024-09-13T12:37:17.017 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction check 2024-09-13T12:37:17.410 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction check succeeded. 2024-09-13T12:37:17.410 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction test 2024-09-13T12:37:17.685 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction test succeeded. 2024-09-13T12:37:17.685 INFO:teuthology.orchestra.run.smithi046.stdout:Running transaction 2024-09-13T12:37:18.405 INFO:teuthology.orchestra.run.smithi042.stdout:Transaction test succeeded. 2024-09-13T12:37:18.406 INFO:teuthology.orchestra.run.smithi042.stdout:Running transaction 2024-09-13T12:37:20.258 INFO:teuthology.orchestra.run.smithi046.stdout: Preparing : 1/1 2024-09-13T12:37:20.913 INFO:teuthology.orchestra.run.smithi046.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T12:37:21.008 INFO:teuthology.orchestra.run.smithi046.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T12:37:21.012 INFO:teuthology.orchestra.run.smithi042.stdout: Preparing : 1/1 2024-09-13T12:37:21.699 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-13T12:37:21.782 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T12:37:23.777 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T12:37:23.907 INFO:teuthology.orchestra.run.smithi046.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T12:37:24.600 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T12:37:24.728 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T12:37:29.033 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T12:37:29.074 INFO:teuthology.orchestra.run.smithi046.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:37:29.850 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T12:37:29.896 INFO:teuthology.orchestra.run.smithi042.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:37:34.212 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:37:35.041 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:13.421 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:13.464 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:13.783 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:13.783 INFO:teuthology.orchestra.run.smithi046.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T12:38:13.783 INFO:teuthology.orchestra.run.smithi046.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T12:38:13.784 INFO:teuthology.orchestra.run.smithi046.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T12:38:14.291 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:14.333 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout:Installed: 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T12:38:14.611 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T12:38:14.612 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:38:14.612 INFO:teuthology.orchestra.run.smithi046.stdout:Complete! 2024-09-13T12:38:14.685 INFO:teuthology.orchestra.run.smithi042.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:14.686 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-13T12:38:14.686 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-13T12:38:14.686 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-13T12:38:14.965 DEBUG:teuthology.orchestra.run.smithi046:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T12:38:15.047 INFO:teuthology.orchestra.run.smithi046.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.048 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.048 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.048 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T12:38:15.048 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T12:38:15.048 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-09-13T12:38:15.048 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi046.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T12:38:15.049 DEBUG:teuthology.orchestra.run.smithi046:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T12:38:15.107 INFO:teuthology.orchestra.run.smithi046.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.107 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.107 DEBUG:teuthology.orchestra.run.smithi046:> sudo rpm -qi grub2-tools 2024-09-13T12:38:15.145 INFO:teuthology.orchestra.run.smithi046.stdout:Name : grub2-tools 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Epoch : 1 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Version : 2.06 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Release : 82.el9 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Architecture: x86_64 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Group : Unspecified 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Size : 8274670 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:License : GPLv3+ 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T12:38:15.146 INFO:teuthology.orchestra.run.smithi046.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:Packager : builder@centos.org 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:Vendor : CentOS 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:Summary : Support tools for GRUB. 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:Description : 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:hardware devices. 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:38:15.147 INFO:teuthology.orchestra.run.smithi046.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T12:38:15.148 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T12:38:15.149 INFO:teuthology.task.kernel:Updating grub on smithi046 to boot 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.149 DEBUG:teuthology.orchestra.run.smithi046:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T12:38:15.487 INFO:teuthology.orchestra.run.smithi042.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout:Installed: 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:38:15.488 INFO:teuthology.orchestra.run.smithi042.stdout:Complete! 2024-09-13T12:38:15.823 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T12:38:15.856 INFO:teuthology.orchestra.run.smithi046.stderr:Generating grub configuration file ... 2024-09-13T12:38:15.908 INFO:teuthology.orchestra.run.smithi042.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.908 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.908 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.908 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T12:38:15.908 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T12:38:15.909 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-09-13T12:38:15.909 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi042.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T12:38:15.909 DEBUG:teuthology.orchestra.run.smithi042:> rpm -q kernel | sort -rV | head -n 1 2024-09-13T12:38:15.967 INFO:teuthology.orchestra.run.smithi042.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-13T12:38:15.967 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-13T12:38:15.967 DEBUG:teuthology.orchestra.run.smithi042:> sudo rpm -qi grub2-tools 2024-09-13T12:38:16.008 INFO:teuthology.orchestra.run.smithi042.stdout:Name : grub2-tools 2024-09-13T12:38:16.008 INFO:teuthology.orchestra.run.smithi042.stdout:Epoch : 1 2024-09-13T12:38:16.008 INFO:teuthology.orchestra.run.smithi042.stdout:Version : 2.06 2024-09-13T12:38:16.008 INFO:teuthology.orchestra.run.smithi042.stdout:Release : 82.el9 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Architecture: x86_64 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Group : Unspecified 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Size : 8274670 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:License : GPLv3+ 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Packager : builder@centos.org 2024-09-13T12:38:16.009 INFO:teuthology.orchestra.run.smithi042.stdout:Vendor : CentOS 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:Summary : Support tools for GRUB. 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:Description : 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:hardware devices. 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:38:16.010 INFO:teuthology.orchestra.run.smithi042.stdout:This subpackage provides tools for support of all platforms. 2024-09-13T12:38:16.011 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-13T12:38:16.011 INFO:teuthology.task.kernel:Updating grub on smithi042 to boot 5.14.0-505.el9.x86_64 2024-09-13T12:38:16.011 DEBUG:teuthology.orchestra.run.smithi042:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-13T12:38:16.678 INFO:teuthology.orchestra.run.smithi042.stderr:Generating grub configuration file ... 2024-09-13T12:38:17.276 INFO:teuthology.orchestra.run.smithi046.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T12:38:17.300 INFO:teuthology.orchestra.run.smithi046.stderr:done 2024-09-13T12:38:17.302 DEBUG:teuthology.orchestra.run.smithi046:> mktemp 2024-09-13T12:38:17.357 INFO:teuthology.orchestra.run.smithi046.stdout:/tmp/tmp.pxsnwaU9tI 2024-09-13T12:38:17.357 DEBUG:teuthology.orchestra.run.smithi046:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.pxsnwaU9tI 2024-09-13T12:38:17.423 DEBUG:teuthology.orchestra.run.smithi046:> sudo chmod 0666 /tmp/tmp.pxsnwaU9tI 2024-09-13T12:38:17.558 DEBUG:teuthology.orchestra.remote:smithi046:/tmp/tmp.pxsnwaU9tI is 6KB 2024-09-13T12:38:17.608 DEBUG:teuthology.orchestra.run.smithi046:> rm -fr /tmp/tmp.pxsnwaU9tI 2024-09-13T12:38:17.622 DEBUG:teuthology.orchestra.run.smithi046:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T12:38:17.683 INFO:teuthology.orchestra.run.smithi046.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T12:38:17.684 INFO:teuthology.orchestra.run.smithi046.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T12:38:17.684 INFO:teuthology.orchestra.run.smithi046.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T12:38:17.684 INFO:teuthology.orchestra.run.smithi046.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T12:38:17.685 DEBUG:teuthology.orchestra.run.smithi046:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T12:38:17.832 DEBUG:teuthology.orchestra.run.smithi046:> sudo shutdown -r now 2024-09-13T12:38:17.969 INFO:teuthology.orchestra.run.smithi042.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-13T12:38:17.997 INFO:teuthology.orchestra.run.smithi042.stderr:done 2024-09-13T12:38:17.999 DEBUG:teuthology.orchestra.run.smithi042:> mktemp 2024-09-13T12:38:18.014 INFO:teuthology.orchestra.run.smithi042.stdout:/tmp/tmp.xxnO2dYO6c 2024-09-13T12:38:18.015 DEBUG:teuthology.orchestra.run.smithi042:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.xxnO2dYO6c 2024-09-13T12:38:18.080 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 0666 /tmp/tmp.xxnO2dYO6c 2024-09-13T12:38:18.223 DEBUG:teuthology.orchestra.remote:smithi042:/tmp/tmp.xxnO2dYO6c is 6KB 2024-09-13T12:38:18.271 DEBUG:teuthology.orchestra.run.smithi042:> rm -fr /tmp/tmp.xxnO2dYO6c 2024-09-13T12:38:18.286 DEBUG:teuthology.orchestra.run.smithi042:> sudo /bin/ls /boot/loader/entries || true 2024-09-13T12:38:18.349 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-13T12:38:18.349 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-13T12:38:18.349 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-13T12:38:18.349 INFO:teuthology.orchestra.run.smithi042.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-13T12:38:18.350 DEBUG:teuthology.orchestra.run.smithi042:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-13T12:38:18.482 DEBUG:teuthology.orchestra.run.smithi042:> sudo shutdown -r now 2024-09-13T12:38:47.865 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-09-13T12:38:47.866 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:38:47.867 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:38:48.486 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-09-13T12:38:48.486 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T12:38:48.487 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:39:06.306 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.46 2024-09-13T12:39:06.813 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.42 2024-09-13T12:39:15.313 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:39:15.314 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:39:15.815 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T12:39:15.815 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:39:18.397 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.46 2024-09-13T12:39:30.409 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:39:30.410 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:39:48.797 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.46 2024-09-13T12:40:03.813 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:40:03.814 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:40:04.409 DEBUG:teuthology.orchestra.run.smithi046:> true 2024-09-13T12:40:04.825 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:40:04.825 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-505.el9.x86_64"... 2024-09-13T12:40:04.825 DEBUG:teuthology.orchestra.run.smithi046:> uname -r 2024-09-13T12:40:04.840 INFO:teuthology.orchestra.run.smithi046.stdout:5.14.0-505.el9.x86_64 2024-09-13T12:40:04.841 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T12:40:04.841 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T12:40:04.841 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T12:40:05.841 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-09-13T12:40:05.843 DEBUG:teuthology.orchestra.run.smithi046:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T12:40:05.940 INFO:teuthology.orchestra.run.smithi046.stdout:ttyS1 2024-09-13T12:40:05.964 DEBUG:teuthology.parallel:result is None 2024-09-13T12:40:15.826 DEBUG:teuthology.orchestra.remote:timed out 2024-09-13T12:40:27.828 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T12:40:27.829 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:40:28.145 DEBUG:teuthology.orchestra.run.smithi042:> true 2024-09-13T12:40:28.516 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T12:40:28.516 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-505.el9.x86_64"... 2024-09-13T12:40:28.516 DEBUG:teuthology.orchestra.run.smithi042:> uname -r 2024-09-13T12:40:28.533 INFO:teuthology.orchestra.run.smithi042.stdout:5.14.0-505.el9.x86_64 2024-09-13T12:40:28.533 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-13T12:40:28.533 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T12:40:28.533 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-13T12:40:29.533 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-09-13T12:40:29.534 DEBUG:teuthology.orchestra.run.smithi042:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T12:40:29.627 INFO:teuthology.orchestra.run.smithi042.stdout:ttyS1 2024-09-13T12:40:29.644 DEBUG:teuthology.parallel:result is None 2024-09-13T12:40:29.644 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T12:40:29.650 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T12:40:29.651 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T12:40:29.653 DEBUG:teuthology.orchestra.run.smithi046:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T12:40:29.675 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T12:40:29.687 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T12:40:29.737 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T12:40:29.737 DEBUG:teuthology.orchestra.run.smithi042:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T12:40:29.740 DEBUG:teuthology.orchestra.run.smithi046:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T12:40:29.772 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T12:40:29.787 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T12:40:29.787 DEBUG:teuthology.orchestra.run.smithi042:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T12:40:29.804 DEBUG:teuthology.orchestra.run.smithi046:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T12:40:29.839 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T12:40:29.848 INFO:teuthology.orchestra.run.smithi046.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T12:40:29.850 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T12:40:29.860 INFO:teuthology.orchestra.run.smithi046.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T12:40:29.861 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T12:40:29.867 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T12:40:29.868 DEBUG:teuthology.orchestra.run.smithi042:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T12:40:29.895 DEBUG:teuthology.orchestra.run.smithi046:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T12:40:29.930 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T12:40:29.939 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T12:40:29.940 DEBUG:teuthology.orchestra.run.smithi042:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T12:40:29.964 DEBUG:teuthology.orchestra.run.smithi046:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T12:40:29.988 DEBUG:teuthology.orchestra.run.smithi042:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T12:40:30.078 DEBUG:teuthology.orchestra.run.smithi042:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T12:40:30.117 DEBUG:teuthology.orchestra.run.smithi042:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T12:40:30.209 DEBUG:teuthology.orchestra.run.smithi042:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T12:40:30.245 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:40:30.245 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T12:40:30.313 DEBUG:teuthology.orchestra.run.smithi046:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T12:40:30.360 DEBUG:teuthology.orchestra.run.smithi046:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T12:40:30.420 DEBUG:teuthology.orchestra.run.smithi046:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T12:40:30.513 DEBUG:teuthology.orchestra.run.smithi046:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T12:40:30.551 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-09-13T12:40:30.551 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T12:40:30.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo service rsyslog restart 2024-09-13T12:40:30.624 DEBUG:teuthology.orchestra.run.smithi046:> sudo service rsyslog restart 2024-09-13T12:40:30.669 INFO:teuthology.orchestra.run.smithi042.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T12:40:30.712 INFO:teuthology.orchestra.run.smithi046.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T12:40:31.153 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T12:40:31.161 INFO:teuthology.task.internal:Starting timer... 2024-09-13T12:40:31.161 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T12:40:31.191 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T12:40:31.242 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-13T12:40:31.242 DEBUG:teuthology.orchestra.run.smithi042:> sudo service auditd rotate 2024-09-13T12:40:31.295 INFO:teuthology.orchestra.run.smithi042.stdout:Rotating logs: 2024-09-13T12:40:31.297 DEBUG:teuthology.orchestra.run.smithi046:> sudo service auditd rotate 2024-09-13T12:40:31.361 INFO:teuthology.orchestra.run.smithi046.stdout:Rotating logs: 2024-09-13T12:40:31.363 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T12:40:31.363 DEBUG:teuthology.orchestra.run.smithi042:> /usr/sbin/getenforce 2024-09-13T12:40:31.386 INFO:teuthology.orchestra.run.smithi042.stdout:Permissive 2024-09-13T12:40:31.386 DEBUG:teuthology.orchestra.run.smithi046:> /usr/sbin/getenforce 2024-09-13T12:40:31.411 INFO:teuthology.orchestra.run.smithi046.stdout:Permissive 2024-09-13T12:40:31.411 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi042.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi046.front.sepia.ceph.com': 'permissive'} 2024-09-13T12:40:31.411 DEBUG:teuthology.orchestra.run.smithi042:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T12:40:31.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T12:40:31.458 DEBUG:teuthology.orchestra.run.smithi046:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T12:40:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T12:40:31.489 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T12:40:31.489 DEBUG:teuthology.orchestra.run.smithi042:> sudo /usr/sbin/setenforce permissive 2024-09-13T12:40:31.535 DEBUG:teuthology.orchestra.run.smithi046:> sudo /usr/sbin/setenforce permissive 2024-09-13T12:40:31.570 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T12:40:31.578 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T12:40:31.590 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T12:40:31.591 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi042.front.sepia.ceph.com,smithi046.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T12:44:23.607 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi042.front.sepia.ceph.com'), Remote(name='ubuntu@smithi046.front.sepia.ceph.com')] 2024-09-13T12:44:23.608 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T12:44:23.610 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi042.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:44:23.702 DEBUG:teuthology.orchestra.run.smithi042:> true 2024-09-13T12:44:23.786 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi042.front.sepia.ceph.com' 2024-09-13T12:44:23.786 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:44:23.787 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T12:44:23.870 DEBUG:teuthology.orchestra.run.smithi046:> true 2024-09-13T12:44:23.950 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-09-13T12:44:23.950 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T12:44:23.961 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T12:44:23.962 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T12:44:23.962 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T12:44:23.965 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T12:44:23.965 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T12:44:23.996 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T12:44:24.011 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T12:44:24.024 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-13T12:44:24.038 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-13T12:44:24.039 INFO:teuthology.orchestra.run.smithi042.stderr:sudo: ntpd: command not found 2024-09-13T12:44:24.051 INFO:teuthology.orchestra.run.smithi042.stdout:506 Cannot talk to daemon 2024-09-13T12:44:24.065 INFO:teuthology.orchestra.run.smithi046.stderr:sudo: ntpd: command not found 2024-09-13T12:44:24.066 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T12:44:24.077 INFO:teuthology.orchestra.run.smithi046.stdout:506 Cannot talk to daemon 2024-09-13T12:44:24.082 INFO:teuthology.orchestra.run.smithi042.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T12:44:24.092 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-13T12:44:24.107 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-13T12:44:24.137 INFO:teuthology.orchestra.run.smithi042.stderr:bash: line 1: ntpq: command not found 2024-09-13T12:44:24.141 INFO:teuthology.orchestra.run.smithi042.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T12:44:24.141 INFO:teuthology.orchestra.run.smithi042.stdout:=============================================================================== 2024-09-13T12:44:24.141 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.141 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.141 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.141 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.164 INFO:teuthology.orchestra.run.smithi046.stderr:bash: line 1: ntpq: command not found 2024-09-13T12:44:24.167 INFO:teuthology.orchestra.run.smithi046.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T12:44:24.167 INFO:teuthology.orchestra.run.smithi046.stdout:=============================================================================== 2024-09-13T12:44:24.167 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.167 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.167 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.167 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T12:44:24.168 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T12:44:24.269 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'CEPHADM_REFRESH_FAILED'], 'log-only-match': ['CEPHADM_'], 'sha1': 'b0201b8c79733293453e7f10a10c7fa43119222b'} 2024-09-13T12:44:24.270 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-09-13T12:44:24.270 INFO:tasks.cephadm:Cluster fsid is e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:44:24.271 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T12:44:24.271 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-09-13T12:44:24.271 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi042': '172.21.15.42', 'mon.smithi046': '172.21.15.46'} 2024-09-13T12:44:24.271 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T12:44:24.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo hostname $(hostname -s) 2024-09-13T12:44:24.298 DEBUG:teuthology.orchestra.run.smithi046:> sudo hostname $(hostname -s) 2024-09-13T12:44:24.325 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-09-13T12:44:24.326 DEBUG:teuthology.orchestra.run.smithi042:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T12:44:24.621 INFO:teuthology.orchestra.run.smithi042.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 12:44 /home/ubuntu/cephtest/cephadm 2024-09-13T12:44:24.621 DEBUG:teuthology.orchestra.run.smithi046:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T12:44:24.866 INFO:teuthology.orchestra.run.smithi046.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Sep 13 12:44 /home/ubuntu/cephtest/cephadm 2024-09-13T12:44:24.866 DEBUG:teuthology.orchestra.run.smithi042:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T12:44:24.884 DEBUG:teuthology.orchestra.run.smithi046:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T12:44:24.912 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-09-13T12:44:24.912 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T12:44:24.927 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-09-13T12:44:26.501 INFO:teuthology.orchestra.run.smithi046.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T12:44:26.628 INFO:teuthology.orchestra.run.smithi042.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T12:44:38.403 INFO:teuthology.orchestra.run.smithi046.stdout:{ 2024-09-13T12:44:38.403 INFO:teuthology.orchestra.run.smithi046.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T12:44:38.403 INFO:teuthology.orchestra.run.smithi046.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T12:44:38.403 INFO:teuthology.orchestra.run.smithi046.stdout: "repo_digests": [ 2024-09-13T12:44:38.404 INFO:teuthology.orchestra.run.smithi046.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T12:44:38.404 INFO:teuthology.orchestra.run.smithi046.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T12:44:38.404 INFO:teuthology.orchestra.run.smithi046.stdout: ] 2024-09-13T12:44:38.404 INFO:teuthology.orchestra.run.smithi046.stdout:} 2024-09-13T12:44:38.629 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:44:38.629 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-09-13T12:44:38.629 INFO:teuthology.orchestra.run.smithi042.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-09-13T12:44:38.629 INFO:teuthology.orchestra.run.smithi042.stdout: "repo_digests": [ 2024-09-13T12:44:38.629 INFO:teuthology.orchestra.run.smithi042.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-09-13T12:44:38.629 INFO:teuthology.orchestra.run.smithi042.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-09-13T12:44:38.630 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T12:44:38.630 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:44:38.647 DEBUG:teuthology.orchestra.run.smithi042:> sudo mkdir -p /etc/ceph 2024-09-13T12:44:38.696 DEBUG:teuthology.orchestra.run.smithi046:> sudo mkdir -p /etc/ceph 2024-09-13T12:44:38.724 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 777 /etc/ceph 2024-09-13T12:44:38.750 DEBUG:teuthology.orchestra.run.smithi046:> sudo chmod 777 /etc/ceph 2024-09-13T12:44:38.790 INFO:tasks.cephadm:Writing seed config... 2024-09-13T12:44:38.791 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T12:44:38.791 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T12:44:38.791 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T12:44:38.791 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T12:44:38.791 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T12:44:38.792 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T12:44:38.792 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T12:44:38.792 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T12:44:38.792 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:44:38.793 DEBUG:teuthology.orchestra.run.smithi042:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T12:44:38.809 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = e7c8d5a8-71cd-11ef-bceb-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T12:44:38.809 DEBUG:teuthology.orchestra.run.smithi042:mon.smithi042> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service 2024-09-13T12:44:38.851 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T12:44:38.851 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.42 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T12:44:39.050 INFO:teuthology.orchestra.run.smithi042.stderr:-------------------------------------------------------------------------------- 2024-09-13T12:44:39.050 INFO:teuthology.orchestra.run.smithi042.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', 'e7c8d5a8-71cd-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.42', '--skip-admin-label'] 2024-09-13T12:44:39.073 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: 5.2.2 2024-09-13T12:44:39.075 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying podman|docker is present... 2024-09-13T12:44:39.096 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: 5.2.2 2024-09-13T12:44:39.098 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying lvm2 is present... 2024-09-13T12:44:39.098 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying time synchronization is in place... 2024-09-13T12:44:39.105 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T12:44:39.112 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-09-13T12:44:39.119 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: enabled 2024-09-13T12:44:39.125 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: active 2024-09-13T12:44:39.126 INFO:teuthology.orchestra.run.smithi042.stderr:Unit chronyd.service is enabled and running 2024-09-13T12:44:39.126 INFO:teuthology.orchestra.run.smithi042.stderr:Repeating the final host check... 2024-09-13T12:44:39.147 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: 5.2.2 2024-09-13T12:44:39.149 INFO:teuthology.orchestra.run.smithi042.stderr:podman (/bin/podman) version 5.2.2 is present 2024-09-13T12:44:39.149 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl is present 2024-09-13T12:44:39.149 INFO:teuthology.orchestra.run.smithi042.stderr:lvcreate is present 2024-09-13T12:44:39.155 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T12:44:39.162 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-09-13T12:44:39.169 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: enabled 2024-09-13T12:44:39.175 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: active 2024-09-13T12:44:39.176 INFO:teuthology.orchestra.run.smithi042.stderr:Unit chronyd.service is enabled and running 2024-09-13T12:44:39.176 INFO:teuthology.orchestra.run.smithi042.stderr:Host looks OK 2024-09-13T12:44:39.176 INFO:teuthology.orchestra.run.smithi042.stderr:Cluster fsid: e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:44:39.176 INFO:teuthology.orchestra.run.smithi042.stderr:Acquiring lock 140099300997008 on /run/cephadm/e7c8d5a8-71cd-11ef-bceb-c7b262605968.lock 2024-09-13T12:44:39.176 INFO:teuthology.orchestra.run.smithi042.stderr:Lock 140099300997008 acquired on /run/cephadm/e7c8d5a8-71cd-11ef-bceb-c7b262605968.lock 2024-09-13T12:44:39.177 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying IP 172.21.15.42 port 3300 ... 2024-09-13T12:44:39.177 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying IP 172.21.15.42 port 6789 ... 2024-09-13T12:44:39.177 INFO:teuthology.orchestra.run.smithi042.stderr:Base mon IP is 172.21.15.42, final addrv is [v2:172.21.15.42:3300,v1:172.21.15.42:6789] 2024-09-13T12:44:39.179 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.42 metric 100 2024-09-13T12:44:39.180 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.42 metric 100 2024-09-13T12:44:39.182 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T12:44:39.182 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-13T12:44:39.182 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1797sec hoplimit 64 pref medium 2024-09-13T12:44:39.185 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T12:44:39.185 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-09-13T12:44:39.185 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T12:44:39.185 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-13T12:44:39.185 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:1440/64 scope link 2024-09-13T12:44:39.185 INFO:teuthology.orchestra.run.smithi042.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-09-13T12:44:39.186 INFO:teuthology.orchestra.run.smithi042.stderr:Mon IP `172.21.15.42` is in CIDR network `172.21.0.0/20` 2024-09-13T12:44:39.186 INFO:teuthology.orchestra.run.smithi042.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T12:44:39.187 INFO:teuthology.orchestra.run.smithi042.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-09-13T12:44:39.289 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-09-13T12:44:39.731 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Getting image source signatures 2024-09-13T12:44:39.731 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-09-13T12:44:39.731 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-09-13T12:44:39.731 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-09-13T12:44:39.732 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-09-13T12:44:39.732 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-09-13T12:44:39.732 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T12:44:39.891 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: Writing manifest to image destination 2024-09-13T12:44:40.044 INFO:teuthology.orchestra.run.smithi042.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-09-13T12:44:40.495 INFO:teuthology.orchestra.run.smithi042.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T12:44:40.993 INFO:teuthology.orchestra.run.smithi042.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-09-13T12:44:40.993 INFO:teuthology.orchestra.run.smithi042.stderr:Extracting ceph user uid/gid from container image... 2024-09-13T12:44:41.443 INFO:teuthology.orchestra.run.smithi042.stderr:stat: 167 167 2024-09-13T12:44:41.941 INFO:teuthology.orchestra.run.smithi042.stderr:Creating initial keys... 2024-09-13T12:44:42.431 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph-authtool: AQC6M+RmhDzJERAAss9qnajI6YO5Ge0V4OoMxw== 2024-09-13T12:44:51.827 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph-authtool: AQDDM+RmASw/ARAAmZs/Std8/kgvGT5LHrQCvw== 2024-09-13T12:44:54.737 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph-authtool: AQDGM+RmKyBvJxAAH2U2jQznbBanAFI10MTXnQ== 2024-09-13T12:44:55.527 INFO:teuthology.orchestra.run.smithi042.stderr:Creating initial monmap... 2024-09-13T12:44:55.985 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T12:44:55.985 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-09-13T12:44:55.986 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:44:55.986 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T12:44:56.499 INFO:teuthology.orchestra.run.smithi042.stderr:monmaptool for smithi042 [v2:172.21.15.42:3300,v1:172.21.15.42:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T12:44:56.499 INFO:teuthology.orchestra.run.smithi042.stderr:setting min_mon_release = octopus 2024-09-13T12:44:56.499 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: set fsid to e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:44:56.499 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T12:44:56.499 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:44:56.499 INFO:teuthology.orchestra.run.smithi042.stderr:Creating mon... 2024-09-13T12:44:57.455 INFO:teuthology.orchestra.run.smithi042.stderr:create mon.smithi042 on 2024-09-13T12:44:57.679 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-13T12:44:57.836 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968.target. 2024-09-13T12:44:57.836 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968.target. 2024-09-13T12:44:58.151 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to reset failed state of unit ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service: Unit ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service not loaded. 2024-09-13T12:44:58.159 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968.target.wants/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service → /etc/systemd/system/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@.service. 2024-09-13T12:44:58.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:58 smithi042 podman[23362]: 2024-09-13 12:44:58.777540981 +0000 UTC m=+0.339948460 container start 431d883b219fbc29c8443316dc0084a03196a831d42a4ccc493dfb263a32a5aa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, version=8, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, distribution-scope=public, release=754, architecture=x86_64, GIT_BRANCH=HEAD, vcs-type=git) 2024-09-13T12:44:58.904 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-09-13T12:44:58.910 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-09-13T12:44:58.911 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-09-13T12:44:58.911 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T12:44:58.912 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mon to start... 2024-09-13T12:44:58.912 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mon... 2024-09-13T12:44:59.136 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:58 smithi042 ceph-mon[23382]: mkfs e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:44:59.136 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:58 smithi042 ceph-mon[23382]: mon.smithi042 is new leader, mons smithi042 in quorum (ranks 0) 2024-09-13T12:44:59.136 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:58 smithi042 bash[23362]: 431d883b219fbc29c8443316dc0084a03196a831d42a4ccc493dfb263a32a5aa 2024-09-13T12:44:59.136 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:58 smithi042 systemd[1]: Started Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: cluster: 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: id: e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: services: 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi042 (age 0.609759s) 2024-09-13T12:44:59.888 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr: no daemons active 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: data: 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: pgs: 2024-09-13T12:44:59.889 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:00.156 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:59 smithi042 ceph-mon[23382]: mon.smithi042 is new leader, mons smithi042 in quorum (ranks 0) 2024-09-13T12:45:00.156 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:59 smithi042 ceph-mon[23382]: monmap e1: 1 mons at {smithi042=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0]} 2024-09-13T12:45:00.156 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:59 smithi042 ceph-mon[23382]: fsmap 2024-09-13T12:45:00.156 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:59 smithi042 ceph-mon[23382]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T12:45:00.156 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:59 smithi042 ceph-mon[23382]: mgrmap e1: no daemons active 2024-09-13T12:45:00.156 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:44:59 smithi042 ceph-mon[23382]: from='client.? 172.21.15.42:0/2458551758' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T12:45:00.442 INFO:teuthology.orchestra.run.smithi042.stderr:mon is available 2024-09-13T12:45:00.443 INFO:teuthology.orchestra.run.smithi042.stderr:Assimilating anything we can from ceph.conf... 2024-09-13T12:45:00.937 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:00.937 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [global] 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: fsid = e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.42:3300,v1:172.21.15.42:6789] 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [mgr] 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:00.938 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [osd] 2024-09-13T12:45:00.939 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T12:45:00.939 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T12:45:01.211 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:00 smithi042 ceph-mon[23382]: from='client.? 172.21.15.42:0/2698788233' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T12:45:01.211 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:00 smithi042 ceph-mon[23382]: from='client.? 172.21.15.42:0/2698788233' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T12:45:01.474 INFO:teuthology.orchestra.run.smithi042.stderr:Generating new minimal ceph.conf... 2024-09-13T12:45:02.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:01 smithi042 ceph-mon[23382]: from='client.? 172.21.15.42:0/2443418437' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:45:02.539 INFO:teuthology.orchestra.run.smithi042.stderr:Restarting the monitor... 2024-09-13T12:45:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:02 smithi042 systemd[1]: Stopping Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T12:45:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:02 smithi042 bash[23600]: Error: no container with name or ID "ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon.smithi042" found: no such container 2024-09-13T12:45:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:02 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[23378]: 2024-09-13T12:45:02.754+0000 7f09f19d4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi042 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T12:45:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:02 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[23378]: 2024-09-13T12:45:02.754+0000 7f09f19d4700 -1 mon.smithi042@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T12:45:03.288 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 podman[23612]: 2024-09-13 12:45:03.021324456 +0000 UTC m=+0.415328936 container died 431d883b219fbc29c8443316dc0084a03196a831d42a4ccc493dfb263a32a5aa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, vendor=Red Hat, Inc., version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, distribution-scope=public, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.buildah.version=1.19.8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD) 2024-09-13T12:45:03.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 podman[23612]: 2024-09-13 12:45:03.379087406 +0000 UTC m=+0.773091927 container cleanup 431d883b219fbc29c8443316dc0084a03196a831d42a4ccc493dfb263a32a5aa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, GIT_CLEAN=True, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, architecture=x86_64, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T12:45:03.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 bash[23612]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042 2024-09-13T12:45:03.664 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 bash[23639]: Error: no container with name or ID "ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon.smithi042" found: no such container 2024-09-13T12:45:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 podman[23625]: 2024-09-13 12:45:03.663129401 +0000 UTC m=+0.634706475 container remove 431d883b219fbc29c8443316dc0084a03196a831d42a4ccc493dfb263a32a5aa (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., distribution-scope=public, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, architecture=x86_64, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, vcs-type=git, GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-13T12:45:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service: Deactivated successfully. 2024-09-13T12:45:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 systemd[1]: Stopped Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T12:45:03.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 systemd[1]: Starting Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T12:45:03.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:03 smithi042 podman[23704]: 2024-09-13 12:45:03.812508522 +0000 UTC m=+0.017628899 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 podman[23704]: 2024-09-13 12:45:04.064139944 +0000 UTC m=+0.269260326 container create f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, version=8, vcs-type=git, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , release=754, name=centos-stream, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream) 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 podman[23704]: 2024-09-13 12:45:04.275116769 +0000 UTC m=+0.480237159 container init f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, maintainer=Guillaume Abrioux , io.openshift.expose-services=, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, architecture=x86_64, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 podman[23704]: 2024-09-13 12:45:04.279237466 +0000 UTC m=+0.484357846 container start f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, com.redhat.component=centos-stream-container, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, version=8, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, ceph=True, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, release=754, GIT_BRANCH=HEAD) 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: pidfile_write: ignore empty --pid-file 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: load: jerasure load: lrc 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: RocksDB version: 6.15.5 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T12:45:04.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Compile date Apr 18 2022 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: DB SUMMARY 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: DB Session ID: DSJY8OMC580U6ZQDVAFO 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: CURRENT file: CURRENT 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi042/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi042/store.db: 000010.log size: 82713 ; 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.error_if_exists: 0 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.create_if_missing: 0 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.paranoid_checks: 1 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T12:45:04.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.env: 0x55b67a6c6860 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.fs: Posix File System 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.info_log: 0x55b67c5d7fa0 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.statistics: (nil) 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.use_fsync: 0 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_log_file_size: 0 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T12:45:04.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T12:45:04.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.allow_fallocate: 1 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.use_direct_reads: 0 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.db_log_dir: 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi042/store.db 2024-09-13T12:45:04.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.write_buffer_manager: 0x55b67c6c8330 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T12:45:04.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.unordered_write: 0 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.row_cache: None 2024-09-13T12:45:04.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.wal_filter: None 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.preserve_deletes: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.two_write_queues: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.atomic_flush: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.log_readahead_size: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T12:45:04.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_background_jobs: 2 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_background_compactions: -1 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_subcompactions: 1 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T12:45:04.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_open_files: -1 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_background_flushes: -1 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Compression algorithms supported: 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kZSTD supported: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kXpressCompression supported: 0 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kLZ4Compression supported: 1 2024-09-13T12:45:04.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kBZip2Compression supported: 0 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kZlibCompression supported: 1 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: kSnappyCompression supported: 1 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi042/store.db/MANIFEST-000009 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.merge_operator: 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_filter: None 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b67c5a3d00) 2024-09-13T12:45:04.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: index_type: 0 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: data_block_index_type: 0 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: index_shortening: 1 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: hash_index_allow_collision: 1 2024-09-13T12:45:04.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: checksum: 1 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: no_block_cache: 0 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache: 0x55b67c60e170 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache_options: 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: capacity : 536870912 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: num_shard_bits : 4 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: strict_capacity_limit : 0 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache_compressed: (nil) 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: persistent_cache: (nil) 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_size: 4096 2024-09-13T12:45:04.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_size_deviation: 10 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_restart_interval: 16 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: index_block_restart_interval: 1 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: metadata_block_size: 4096 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: partition_filters: 0 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: use_delta_encoding: 1 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: whole_key_filtering: 1 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: verify_compression: 0 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: format_version: 4 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: enable_index_compression: 1 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_align: 0 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T12:45:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression: NoCompression 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.num_levels: 7 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T12:45:04.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T12:45:04.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T12:45:04.375 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T12:45:04.376 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.table_properties_collectors: 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.inplace_update_support: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.bloom_locality: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.max_successive_merges: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.ttl: 2592000 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.enable_blob_files: false 2024-09-13T12:45:04.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.min_blob_size: 0 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi042/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726231504308431, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726231504309267, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79395, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77659, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71968, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726231504, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "068c3d00-9ff0-40eb-85f3-10586e31a93a", "db_session_id": "DSJY8OMC580U6ZQDVAFO"}} 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726231504309650, "job": 1, "event": "recovery_finished"} 2024-09-13T12:45:04.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi042/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55b67c5f4a80 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: DB pointer 0x55b67c604000 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** DB Stats ** 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T12:45:04.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T12:45:04.380 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative compaction: 0.00 GB write, 21.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval compaction: 0.00 GB write, 21.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:45:04.381 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: L0 2/0 79.33 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Sum 2/0 79.33 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 134.7 0.00 0.00 1 0.001 0 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T12:45:04.382 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative compaction: 0.00 GB write, 21.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: starting mon.smithi042 rank 0 at public addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] at bind addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi042 fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???) e1 preinit fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).mds e1 new map 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).mds e1 print_map 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: e1 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: legacy client fscid: -1 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T12:45:04.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: No filesystems configured 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 bash[23704]: f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 2024-09-13T12:45:04.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 systemd[1]: Started Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T12:45:04.384 INFO:teuthology.orchestra.run.smithi042.stderr:Setting mon public_network to 172.21.0.0/20 2024-09-13T12:45:04.661 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mon.smithi042 is new leader, mons smithi042 in quorum (ranks 0) 2024-09-13T12:45:04.661 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: monmap e1: 1 mons at {smithi042=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0]} 2024-09-13T12:45:04.662 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: fsmap 2024-09-13T12:45:04.662 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: osdmap e1: 0 total, 0 up, 0 in 2024-09-13T12:45:04.662 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:04 smithi042 ceph-mon[23723]: mgrmap e1: no daemons active 2024-09-13T12:45:05.702 INFO:teuthology.orchestra.run.smithi042.stderr:Wrote config to /etc/ceph/ceph.conf 2024-09-13T12:45:05.702 INFO:teuthology.orchestra.run.smithi042.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T12:45:05.702 INFO:teuthology.orchestra.run.smithi042.stderr:Creating mgr... 2024-09-13T12:45:05.703 INFO:teuthology.orchestra.run.smithi042.stderr:Verifying port 9283 ... 2024-09-13T12:45:05.889 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Failed to reset failed state of unit ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mgr.smithi042.jhsfyp.service: Unit ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mgr.smithi042.jhsfyp.service not loaded. 2024-09-13T12:45:05.897 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: Created symlink /etc/systemd/system/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968.target.wants/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mgr.smithi042.jhsfyp.service → /etc/systemd/system/ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@.service. 2024-09-13T12:45:05.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:05 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3910717309' entity='client.admin' 2024-09-13T12:45:06.591 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-09-13T12:45:06.598 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-09-13T12:45:06.599 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-09-13T12:45:06.599 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to enable service . firewalld.service is not available 2024-09-13T12:45:06.607 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-09-13T12:45:06.615 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-09-13T12:45:06.616 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-09-13T12:45:06.616 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-09-13T12:45:06.617 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr to start... 2024-09-13T12:45:06.617 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr... 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsid": "e7c8d5a8-71cd-11ef-bceb-c7b262605968", 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "health": { 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T12:45:07.206 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 0 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T12:45:07.207 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "smithi042" 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T12:45:07.208 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.209 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T12:45:07.209 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:07.209 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T12:45:07.209 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T12:45:07.209 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T12:45:07.209 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T12:45:07.213 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": false, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T12:45:07.214 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "iostat", 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "nfs", 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "restful" 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modified": "2024-09-13T12:44:58.814182+0000", 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T12:45:07.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:07.397 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:07 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/362852384' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T12:45:07.744 INFO:teuthology.orchestra.run.smithi042.stderr:mgr not available, waiting (1/15)... 2024-09-13T12:45:10.244 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:10.244 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsid": "e7c8d5a8-71cd-11ef-bceb-c7b262605968", 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "health": { 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 0 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "smithi042" 2024-09-13T12:45:10.245 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T12:45:10.246 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T12:45:10.250 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T12:45:10.251 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": false, 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "iostat", 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "nfs", 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "restful" 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:10.252 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modified": "2024-09-13T12:44:58.814182+0000", 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T12:45:10.253 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:10.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:10 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2004304815' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T12:45:11.208 INFO:teuthology.orchestra.run.smithi042.stderr:mgr not available, waiting (2/15)... 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: Activating manager daemon smithi042.jhsfyp 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: mgrmap e2: smithi042.jhsfyp(active, starting, since 0.00300412s) 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:45:13.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:45:13.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:13.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:13.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:12 smithi042 ceph-mon[23723]: from='mgr.14100 172.21.15.42:0/2501601413' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:13.727 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:13.727 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:13.727 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsid": "e7c8d5a8-71cd-11ef-bceb-c7b262605968", 2024-09-13T12:45:13.727 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "health": { 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 0 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "smithi042" 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:13.728 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T12:45:13.729 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T12:45:13.733 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T12:45:13.733 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T12:45:13.733 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.733 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T12:45:13.733 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T12:45:13.734 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": false, 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "iostat", 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "nfs", 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "restful" 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:13.735 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modified": "2024-09-13T12:44:58.814182+0000", 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T12:45:13.736 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:14.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:13 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3650897070' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T12:45:14.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:13 smithi042 ceph-mon[23723]: mgrmap e3: smithi042.jhsfyp(active, since 1.00653s) 2024-09-13T12:45:14.323 INFO:teuthology.orchestra.run.smithi042.stderr:mgr not available, waiting (3/15)... 2024-09-13T12:45:16.657 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:16 smithi042 ceph-mon[23723]: mgrmap e4: smithi042.jhsfyp(active, since 2s) 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsid": "e7c8d5a8-71cd-11ef-bceb-c7b262605968", 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "health": { 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "checks": {}, 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mutes": [] 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-09-13T12:45:17.026 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum": [ 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 0 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_names": [ 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "smithi042" 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "quorum_age": 12, 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "monmap": { 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_mons": 1 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osdmap": { 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:17.027 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_osds": 0, 2024-09-13T12:45:17.028 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-09-13T12:45:17.028 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-09-13T12:45:17.028 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgmap": { 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_pools": 0, 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_objects": 0, 2024-09-13T12:45:17.031 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "bytes_total": 0 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "fsmap": { 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "by_rank": [], 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "up:standby": 0 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap": { 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": true, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modules": [ 2024-09-13T12:45:17.032 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "iostat", 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "nfs", 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "restful" 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ], 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "servicemap": { 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 1, 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "modified": "2024-09-13T12:44:58.814182+0000", 2024-09-13T12:45:17.033 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "services": {} 2024-09-13T12:45:17.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: }, 2024-09-13T12:45:17.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "progress_events": {} 2024-09-13T12:45:17.034 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:17.585 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:17 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3585682475' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T12:45:17.588 INFO:teuthology.orchestra.run.smithi042.stderr:mgr is available 2024-09-13T12:45:18.180 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:18.180 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [global] 2024-09-13T12:45:18.180 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: fsid = e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [mgr] 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: [osd] 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-09-13T12:45:18.181 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-09-13T12:45:18.485 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:18 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/557097170' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T12:45:18.761 INFO:teuthology.orchestra.run.smithi042.stderr:Enabling cephadm module... 2024-09-13T12:45:20.582 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:20 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/4264897655' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T12:45:21.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:21 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/4264897655' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T12:45:21.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:21 smithi042 ceph-mon[23723]: mgrmap e5: smithi042.jhsfyp(active, since 7s) 2024-09-13T12:45:21.613 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:21.613 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 5, 2024-09-13T12:45:21.614 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": true, 2024-09-13T12:45:21.614 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "active_name": "smithi042.jhsfyp", 2024-09-13T12:45:21.614 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T12:45:21.614 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:22.206 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for the mgr to restart... 2024-09-13T12:45:22.206 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr epoch 5... 2024-09-13T12:45:22.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:22 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3468531762' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T12:45:27.808 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:27.809 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-09-13T12:45:27.809 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T12:45:27.809 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:28.089 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: Active manager daemon smithi042.jhsfyp restarted 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: Activating manager daemon smithi042.jhsfyp 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: osdmap e2: 0 total, 0 up, 0 in 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: mgrmap e6: smithi042.jhsfyp(active, starting, since 0.0525073s) 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:28.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:28.091 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:28.091 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:28.091 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:45:28.091 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:27 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:45:28.463 INFO:teuthology.orchestra.run.smithi042.stderr:mgr epoch 5 is available 2024-09-13T12:45:28.463 INFO:teuthology.orchestra.run.smithi042.stderr:Setting orchestrator backend to cephadm... 2024-09-13T12:45:29.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:28 smithi042 ceph-mon[23723]: mgrmap e7: smithi042.jhsfyp(active, since 1.05467s) 2024-09-13T12:45:29.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:28 smithi042 ceph-mon[23723]: [13/Sep/2024:12:45:27] ENGINE Bus STARTING 2024-09-13T12:45:29.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:28 smithi042 ceph-mon[23723]: [13/Sep/2024:12:45:28] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T12:45:29.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:28 smithi042 ceph-mon[23723]: [13/Sep/2024:12:45:28] ENGINE Bus STARTED 2024-09-13T12:45:29.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:28 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:29.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:28 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:30.310 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:30 smithi042 ceph-mon[23723]: mgrmap e8: smithi042.jhsfyp(active, since 2s) 2024-09-13T12:45:30.310 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:30 smithi042 ceph-mon[23723]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:30.310 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:30 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:30.310 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:30 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:30.310 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: value unchanged 2024-09-13T12:45:30.925 INFO:teuthology.orchestra.run.smithi042.stderr:Generating ssh key... 2024-09-13T12:45:31.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:31 smithi042 ceph-mon[23723]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:32.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:32 smithi042 ceph-mon[23723]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:32.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:32 smithi042 ceph-mon[23723]: Generating ssh key... 2024-09-13T12:45:33.730 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:33 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:33.730 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:33 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:33.730 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDd76/8vJgXKSioqGI+OKk1uLMs8PFwFnryQ2bjGWMhdo11it9tZa4E/S0phC+WPQiYA6Sb04U9KYHTxlv+LzfRP4667T561COcSXbs0i4y78Ju4tYxMoZSDYsc4XgYENQSmyS6fKGVg4twPmfPtyuLl6JffnNVSecKm/xW7PJGggjLLOyQeT1KYlk4SXAcyl0BoNr7sRWD5rAnv5VIY5xkHZ4aZvrjIKCOZshQCNCL69cqPjixTlr0DJlotPrSmaf1M1dXue6WN+86g6bGFucEcCQer7ApScKdE4C9aoy96wnHqZWaMxZhu8hKQPIDrcaXDzC3OUslTzJ9oyooYisEBCdPIgK+6mCz9nCLzOJ6Y7xh+a56ignOH1RMkYFN1fvbc18hJvwlapLm3XzYOsyTOeoJ76yqnDsd/ohlY20nCbd4ILsYJUMD6+IAJLE78/zqg2dRNcQuz73L8Q7s1wiXV2luVOaqV3oBN4A4kkypSaoy7zECINn893YyBkx+0Pk= ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:45:34.246 INFO:teuthology.orchestra.run.smithi042.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T12:45:34.246 INFO:teuthology.orchestra.run.smithi042.stderr:Adding key to root@localhost authorized_keys... 2024-09-13T12:45:34.253 INFO:teuthology.orchestra.run.smithi042.stderr:Adding host smithi042... 2024-09-13T12:45:35.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:35 smithi042 ceph-mon[23723]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:35.793 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Added host 'smithi042' with addr '172.21.15.42' 2024-09-13T12:45:36.352 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying mon service with default placement... 2024-09-13T12:45:36.451 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:36 smithi042 ceph-mon[23723]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi042", "addr": "172.21.15.42", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:36.451 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:36 smithi042 ceph-mon[23723]: Deploying cephadm binary to smithi042 2024-09-13T12:45:36.451 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:36 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:36.451 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:36 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:36.974 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled mon update... 2024-09-13T12:45:37.609 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:37 smithi042 ceph-mon[23723]: Added host smithi042 2024-09-13T12:45:37.610 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:37 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:37.611 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying mgr service with default placement... 2024-09-13T12:45:38.215 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-09-13T12:45:38.475 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:38 smithi042 ceph-mon[23723]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:38.475 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:38 smithi042 ceph-mon[23723]: Saving service mon spec with placement count:5 2024-09-13T12:45:38.475 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:38 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:38.829 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying crash service with default placement... 2024-09-13T12:45:39.449 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:39 smithi042 ceph-mon[23723]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:39.449 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:39 smithi042 ceph-mon[23723]: Saving service mgr spec with placement count:2 2024-09-13T12:45:40.460 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled crash update... 2024-09-13T12:45:40.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:40 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:40.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:40 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:40.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:40 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:41.022 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying prometheus service with default placement... 2024-09-13T12:45:41.971 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:41 smithi042 ceph-mon[23723]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:41.971 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:41 smithi042 ceph-mon[23723]: Saving service crash spec with placement * 2024-09-13T12:45:42.342 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-09-13T12:45:42.992 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying grafana service with default placement... 2024-09-13T12:45:43.206 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:42 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:43.206 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:42 smithi042 ceph-mon[23723]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:43.206 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:42 smithi042 ceph-mon[23723]: Saving service prometheus spec with placement count:1 2024-09-13T12:45:43.206 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:42 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:43.206 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:42 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:43.588 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-09-13T12:45:44.175 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying node-exporter service with default placement... 2024-09-13T12:45:44.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:44 smithi042 ceph-mon[23723]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:44.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:44 smithi042 ceph-mon[23723]: Saving service grafana spec with placement count:1 2024-09-13T12:45:44.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:44 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:44.807 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-09-13T12:45:45.575 INFO:teuthology.orchestra.run.smithi042.stderr:Deploying alertmanager service with default placement... 2024-09-13T12:45:45.950 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:45 smithi042 ceph-mon[23723]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:45.950 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:45 smithi042 ceph-mon[23723]: Saving service node-exporter spec with placement * 2024-09-13T12:45:45.951 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:45 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:46.200 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-09-13T12:45:47.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:47 smithi042 ceph-mon[23723]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:45:47.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:47 smithi042 ceph-mon[23723]: Saving service alertmanager spec with placement count:1 2024-09-13T12:45:47.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:47 smithi042 ceph-mon[23723]: from='mgr.14120 172.21.15.42:0/2547881312' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:48.453 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:48 smithi042 ceph-mon[23723]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:45:48.453 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:48 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3374685758' entity='client.admin' 2024-09-13T12:45:49.523 INFO:teuthology.orchestra.run.smithi042.stderr:Enabling the dashboard module... 2024-09-13T12:45:49.973 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:49 smithi042 ceph-mon[23723]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:45:49.973 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:49 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1162784452' entity='client.admin' 2024-09-13T12:45:51.147 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:50 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2653411729' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T12:45:51.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:51 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2653411729' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T12:45:51.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:51 smithi042 ceph-mon[23723]: mgrmap e9: smithi042.jhsfyp(active, since 24s) 2024-09-13T12:45:52.172 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:52.172 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "epoch": 9, 2024-09-13T12:45:52.172 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "available": true, 2024-09-13T12:45:52.172 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "active_name": "smithi042.jhsfyp", 2024-09-13T12:45:52.172 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "num_standby": 0 2024-09-13T12:45:52.172 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:52.736 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for the mgr to restart... 2024-09-13T12:45:52.736 INFO:teuthology.orchestra.run.smithi042.stderr:Waiting for mgr epoch 9... 2024-09-13T12:45:52.984 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:52 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1807205204' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T12:45:58.311 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: Active manager daemon smithi042.jhsfyp restarted 2024-09-13T12:45:58.311 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: Activating manager daemon smithi042.jhsfyp 2024-09-13T12:45:58.311 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: osdmap e3: 0 total, 0 up, 0 in 2024-09-13T12:45:58.311 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: mgrmap e10: smithi042.jhsfyp(active, starting, since 0.0524526s) 2024-09-13T12:45:58.311 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:45:58.312 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:58 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:45:58.313 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: { 2024-09-13T12:45:58.313 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-09-13T12:45:58.313 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: "initialized": true 2024-09-13T12:45:58.313 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: } 2024-09-13T12:45:59.022 INFO:teuthology.orchestra.run.smithi042.stderr:mgr epoch 9 is available 2024-09-13T12:45:59.022 INFO:teuthology.orchestra.run.smithi042.stderr:Generating a dashboard self-signed certificate... 2024-09-13T12:45:59.539 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:59 smithi042 ceph-mon[23723]: mgrmap e11: smithi042.jhsfyp(active, since 1.0549s) 2024-09-13T12:45:59.539 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:59 smithi042 ceph-mon[23723]: [13/Sep/2024:12:45:58] ENGINE Bus STARTING 2024-09-13T12:45:59.539 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:59 smithi042 ceph-mon[23723]: [13/Sep/2024:12:45:58] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T12:45:59.539 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:59 smithi042 ceph-mon[23723]: [13/Sep/2024:12:45:58] ENGINE Bus STARTED 2024-09-13T12:45:59.539 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:45:59 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:45:59.781 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: Self-signed certificate created 2024-09-13T12:46:00.487 INFO:teuthology.orchestra.run.smithi042.stderr:Creating initial admin user... 2024-09-13T12:46:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:00 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:00 smithi042 ceph-mon[23723]: mgrmap e12: smithi042.jhsfyp(active, since 2s) 2024-09-13T12:46:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:00 smithi042 ceph-mon[23723]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:46:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:00 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:00 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:00 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:01.433 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$05oLuDGdvqe3SYTI..0p2uatk6wqA.755aCm8sOn81r.KuHCt5i0G", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726231561, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T12:46:02.283 INFO:teuthology.orchestra.run.smithi042.stderr:Fetching dashboard port number... 2024-09-13T12:46:02.684 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:02 smithi042 ceph-mon[23723]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:46:02.684 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:02 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:03.435 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: 8443 2024-09-13T12:46:04.061 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: disabled 2024-09-13T12:46:04.069 INFO:teuthology.orchestra.run.smithi042.stderr:systemctl: inactive 2024-09-13T12:46:04.070 INFO:teuthology.orchestra.run.smithi042.stderr:firewalld.service is not enabled 2024-09-13T12:46:04.070 INFO:teuthology.orchestra.run.smithi042.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr:Ceph Dashboard is now available at: 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr: URL: https://smithi042.front.sepia.ceph.com:8443/ 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr: User: admin 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr: Password: 6qqrx57z0t 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:04.073 INFO:teuthology.orchestra.run.smithi042.stderr:Enabling autotune for osd_memory_target 2024-09-13T12:46:04.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:04 smithi042 ceph-mon[23723]: mgrmap e13: smithi042.jhsfyp(active, since 6s) 2024-09-13T12:46:04.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:04 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1651836842' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T12:46:06.053 INFO:teuthology.orchestra.run.smithi042.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-09-13T12:46:06.655 INFO:teuthology.orchestra.run.smithi042.stderr:You can access the Ceph CLI with: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr:Please consider enabling telemetry to help improve Ceph: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: ceph telemetry on 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr:For more information see: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T12:46:06.656 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T12:46:06.657 INFO:teuthology.orchestra.run.smithi042.stderr:Bootstrap complete. 2024-09-13T12:46:06.705 INFO:tasks.cephadm:Fetching config... 2024-09-13T12:46:06.706 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:46:06.706 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T12:46:06.724 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T12:46:06.724 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:46:06.724 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T12:46:06.781 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T12:46:06.781 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:46:06.781 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/keyring of=/dev/stdout 2024-09-13T12:46:06.849 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T12:46:06.849 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:46:06.849 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T12:46:06.906 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T12:46:06.906 DEBUG:teuthology.orchestra.run.smithi042:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDd76/8vJgXKSioqGI+OKk1uLMs8PFwFnryQ2bjGWMhdo11it9tZa4E/S0phC+WPQiYA6Sb04U9KYHTxlv+LzfRP4667T561COcSXbs0i4y78Ju4tYxMoZSDYsc4XgYENQSmyS6fKGVg4twPmfPtyuLl6JffnNVSecKm/xW7PJGggjLLOyQeT1KYlk4SXAcyl0BoNr7sRWD5rAnv5VIY5xkHZ4aZvrjIKCOZshQCNCL69cqPjixTlr0DJlotPrSmaf1M1dXue6WN+86g6bGFucEcCQer7ApScKdE4C9aoy96wnHqZWaMxZhu8hKQPIDrcaXDzC3OUslTzJ9oyooYisEBCdPIgK+6mCz9nCLzOJ6Y7xh+a56ignOH1RMkYFN1fvbc18hJvwlapLm3XzYOsyTOeoJ76yqnDsd/ohlY20nCbd4ILsYJUMD6+IAJLE78/zqg2dRNcQuz73L8Q7s1wiXV2luVOaqV3oBN4A4kkypSaoy7zECINn893YyBkx+0Pk= ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T12:46:06.982 INFO:teuthology.orchestra.run.smithi042.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDd76/8vJgXKSioqGI+OKk1uLMs8PFwFnryQ2bjGWMhdo11it9tZa4E/S0phC+WPQiYA6Sb04U9KYHTxlv+LzfRP4667T561COcSXbs0i4y78Ju4tYxMoZSDYsc4XgYENQSmyS6fKGVg4twPmfPtyuLl6JffnNVSecKm/xW7PJGggjLLOyQeT1KYlk4SXAcyl0BoNr7sRWD5rAnv5VIY5xkHZ4aZvrjIKCOZshQCNCL69cqPjixTlr0DJlotPrSmaf1M1dXue6WN+86g6bGFucEcCQer7ApScKdE4C9aoy96wnHqZWaMxZhu8hKQPIDrcaXDzC3OUslTzJ9oyooYisEBCdPIgK+6mCz9nCLzOJ6Y7xh+a56ignOH1RMkYFN1fvbc18hJvwlapLm3XzYOsyTOeoJ76yqnDsd/ohlY20nCbd4ILsYJUMD6+IAJLE78/zqg2dRNcQuz73L8Q7s1wiXV2luVOaqV3oBN4A4kkypSaoy7zECINn893YyBkx+0Pk= ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:46:06.994 DEBUG:teuthology.orchestra.run.smithi046:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDd76/8vJgXKSioqGI+OKk1uLMs8PFwFnryQ2bjGWMhdo11it9tZa4E/S0phC+WPQiYA6Sb04U9KYHTxlv+LzfRP4667T561COcSXbs0i4y78Ju4tYxMoZSDYsc4XgYENQSmyS6fKGVg4twPmfPtyuLl6JffnNVSecKm/xW7PJGggjLLOyQeT1KYlk4SXAcyl0BoNr7sRWD5rAnv5VIY5xkHZ4aZvrjIKCOZshQCNCL69cqPjixTlr0DJlotPrSmaf1M1dXue6WN+86g6bGFucEcCQer7ApScKdE4C9aoy96wnHqZWaMxZhu8hKQPIDrcaXDzC3OUslTzJ9oyooYisEBCdPIgK+6mCz9nCLzOJ6Y7xh+a56ignOH1RMkYFN1fvbc18hJvwlapLm3XzYOsyTOeoJ76yqnDsd/ohlY20nCbd4ILsYJUMD6+IAJLE78/zqg2dRNcQuz73L8Q7s1wiXV2luVOaqV3oBN4A4kkypSaoy7zECINn893YyBkx+0Pk= ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T12:46:07.046 INFO:teuthology.orchestra.run.smithi046.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDd76/8vJgXKSioqGI+OKk1uLMs8PFwFnryQ2bjGWMhdo11it9tZa4E/S0phC+WPQiYA6Sb04U9KYHTxlv+LzfRP4667T561COcSXbs0i4y78Ju4tYxMoZSDYsc4XgYENQSmyS6fKGVg4twPmfPtyuLl6JffnNVSecKm/xW7PJGggjLLOyQeT1KYlk4SXAcyl0BoNr7sRWD5rAnv5VIY5xkHZ4aZvrjIKCOZshQCNCL69cqPjixTlr0DJlotPrSmaf1M1dXue6WN+86g6bGFucEcCQer7ApScKdE4C9aoy96wnHqZWaMxZhu8hKQPIDrcaXDzC3OUslTzJ9oyooYisEBCdPIgK+6mCz9nCLzOJ6Y7xh+a56ignOH1RMkYFN1fvbc18hJvwlapLm3XzYOsyTOeoJ76yqnDsd/ohlY20nCbd4ILsYJUMD6+IAJLE78/zqg2dRNcQuz73L8Q7s1wiXV2luVOaqV3oBN4A4kkypSaoy7zECINn893YyBkx+0Pk= ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:46:07.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T12:46:07.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:07 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1170163343' entity='client.admin' 2024-09-13T12:46:09.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:09 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:09.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:09 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi042", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:46:09.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:09 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:09.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:09 smithi042 ceph-mon[23723]: Deploying daemon alertmanager.smithi042 on smithi042 2024-09-13T12:46:10.530 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T12:46:10.530 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T12:46:11.098 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:10 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3366285569' entity='client.admin' 2024-09-13T12:46:14.123 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi046 2024-09-13T12:46:14.123 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-09-13T12:46:14.123 DEBUG:teuthology.orchestra.run.smithi046:> dd of=/etc/ceph/ceph.conf 2024-09-13T12:46:14.143 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-09-13T12:46:14.143 DEBUG:teuthology.orchestra.run.smithi046:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:46:14.201 INFO:tasks.cephadm:Adding host smithi046 to orchestrator... 2024-09-13T12:46:14.201 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch host add smithi046 2024-09-13T12:46:14.229 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:13 smithi042 ceph-mon[23723]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:46:14.230 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:13 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:17.201 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:16 smithi042 ceph-mon[23723]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi046", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:46:17.768 INFO:teuthology.orchestra.run.smithi042.stdout:Added host 'smithi046' with addr '172.21.15.46' 2024-09-13T12:46:18.517 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T12:46:19.052 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:18 smithi042 ceph-mon[23723]: Deploying cephadm binary to smithi046 2024-09-13T12:46:19.052 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:18 smithi042 ceph-mon[23723]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:19.052 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:18 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:19.052 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:18 smithi042 ceph-mon[23723]: Added host smithi046 2024-09-13T12:46:20.915 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:46:20.915 INFO:teuthology.orchestra.run.smithi042.stdout:[{"addr": "172.21.15.42", "hostname": "smithi042", "labels": [], "status": ""}, {"addr": "172.21.15.46", "hostname": "smithi046", "labels": [], "status": ""}] 2024-09-13T12:46:20.916 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:20 smithi042 ceph-mon[23723]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:21.952 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:21 smithi042 ceph-mon[23723]: from='client.14190 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:46:21.976 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T12:46:21.976 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T12:46:22.925 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:22 smithi042 ceph-mon[23723]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:24.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:24 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:24.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:24 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:46:24.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:24 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T12:46:24.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:24 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:46:24.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:24 smithi042 ceph-mon[23723]: Deploying daemon crash.smithi042 on smithi042 2024-09-13T12:46:24.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:24 smithi042 ceph-mon[23723]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:25.335 INFO:teuthology.orchestra.run.smithi042.stderr:adjusted tunables profile to default 2024-09-13T12:46:25.712 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:25 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2164024316' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T12:46:26.275 INFO:tasks.cephadm:Adding mon.smithi042 on smithi042 2024-09-13T12:46:26.275 INFO:tasks.cephadm:Adding mon.smithi046 on smithi046 2024-09-13T12:46:26.276 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch apply mon '2;smithi042:172.21.15.42=smithi042;smithi046:172.21.15.46=smithi046' 2024-09-13T12:46:26.593 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:26 smithi042 ceph-mon[23723]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:26.594 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:26 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2164024316' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T12:46:26.594 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:26 smithi042 ceph-mon[23723]: osdmap e4: 0 total, 0 up, 0 in 2024-09-13T12:46:27.931 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled mon update... 2024-09-13T12:46:28.246 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:28.319 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:28.319 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:28.319 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T12:46:28.319 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T12:46:28.319 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:28.320 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: Deploying daemon grafana.smithi042 on smithi042 2024-09-13T12:46:28.320 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:27 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:28.502 DEBUG:teuthology.orchestra.run.smithi046:mon.smithi046> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi046.service 2024-09-13T12:46:28.505 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:28.505 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:29.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:28 smithi042 ceph-mon[23723]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:29.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:28 smithi042 ceph-mon[23723]: from='client.14194 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi042:172.21.15.42=smithi042;smithi046:172.21.15.46=smithi046", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:46:29.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:28 smithi042 ceph-mon[23723]: Saving service mon spec with placement smithi042:172.21.15.42=smithi042;smithi046:172.21.15.46=smithi046;count:2 2024-09-13T12:46:29.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:28 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:46:30.264 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:30.265 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:30.266 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:31.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:30 smithi042 ceph-mon[23723]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:31.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:30 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1596684982' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:31.851 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:31.851 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:33.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:32 smithi042 ceph-mon[23723]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:33.711 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:33.711 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:33.713 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:34.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:33 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/444770386' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:35.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:34 smithi042 ceph-mon[23723]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:35.305 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:35.305 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:37.171 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:37.171 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:37.173 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:37.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:36 smithi042 ceph-mon[23723]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:38.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:37 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3740740161' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:38.773 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:38.773 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:39.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:38 smithi042 ceph-mon[23723]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:40.690 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:40.690 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:40.692 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:41.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:40 smithi042 ceph-mon[23723]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:41.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:40 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1743910158' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:42.173 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:42.173 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:43.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:42 smithi042 ceph-mon[23723]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:44.081 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:44.081 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:44.083 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:45.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:44 smithi042 ceph-mon[23723]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:44 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3434500403' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:45.658 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:45.659 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:47.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:46 smithi042 ceph-mon[23723]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:47.387 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:47.387 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:47.389 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:48.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:47 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1470428726' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:48.877 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:48.878 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:49.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:48 smithi042 ceph-mon[23723]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:50.816 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:50.816 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:50.818 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:51.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:50 smithi042 ceph-mon[23723]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:51.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:50 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/947449867' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:52.356 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:52.356 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:53.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:52 smithi042 ceph-mon[23723]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:54.201 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:54.201 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:54.203 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:54.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:54 smithi042 ceph-mon[23723]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:55 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3864622970' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:55.750 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:55.751 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:46:56.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:56 smithi042 ceph-mon[23723]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:57.549 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:46:57.549 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:46:57.551 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:46:57.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:57 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:46:57.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:57 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:46:58.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:58 smithi042 ceph-mon[23723]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:46:58.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:46:58 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/487694566' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:46:59.143 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:46:59.193 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:00.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:00 smithi042 ceph-mon[23723]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:01.012 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:01.012 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:01.014 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:01.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:01 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2270428046' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:02.556 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:02.556 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:02.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:02 smithi042 ceph-mon[23723]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:04.462 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:04.462 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:04.464 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:04 smithi042 ceph-mon[23723]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:05.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:05 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1167015739' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:06.238 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:06.239 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:06.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:06 smithi042 ceph-mon[23723]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:08.122 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:08.122 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:08.124 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:08.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:08 smithi042 ceph-mon[23723]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:08.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:08 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2059197243' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:09.656 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:09.657 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:10 smithi042 ceph-mon[23723]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:11.517 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:11.517 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:11.518 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:12 smithi042 ceph-mon[23723]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:12 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/242737659' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:13.058 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:13.058 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:14.741 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:14.741 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:14.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:14 smithi042 ceph-mon[23723]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:14.743 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:15.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:15 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/4139872331' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:16.286 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:16.286 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:16.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:16 smithi042 ceph-mon[23723]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:18.229 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:18.229 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:18.231 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:18.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:18 smithi042 ceph-mon[23723]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:18 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/609371224' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:19.807 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:19.808 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:20.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:20 smithi042 ceph-mon[23723]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:21.579 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:21.579 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:21.581 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:22.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:22 smithi042 ceph-mon[23723]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:22.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:22 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2659692317' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:23.214 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:23.215 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:24.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:24 smithi042 ceph-mon[23723]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:25.085 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:25.085 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:25.087 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:25.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:25 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/211041732' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:26.623 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:26.623 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:26.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:26 smithi042 ceph-mon[23723]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:28.459 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:28.459 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:28.461 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:28.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:28 smithi042 ceph-mon[23723]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:29.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:29 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1578540323' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:30.087 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:30.087 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:30.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:30 smithi042 ceph-mon[23723]: pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:31.982 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:31.983 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:31.985 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:32.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:32 smithi042 ceph-mon[23723]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:32.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:32 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/36279818' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:33.655 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:33.656 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:34.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:34 smithi042 ceph-mon[23723]: pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:35.479 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:35.480 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:35.481 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:36.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:36 smithi042 ceph-mon[23723]: pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:36 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3707562484' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:37.104 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:37.104 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:38.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:38 smithi042 ceph-mon[23723]: pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:38.995 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:38.995 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:38.997 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:39.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:39 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/318690523' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:40.579 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:40.580 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:40.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:40 smithi042 ceph-mon[23723]: pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:42.412 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:42.412 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:42.414 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:42.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:42 smithi042 ceph-mon[23723]: pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:42.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:42 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/870799198' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:44.141 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:44.142 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:44.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:44 smithi042 ceph-mon[23723]: pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:46.006 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:46.007 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:46.009 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:46.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:46 smithi042 ceph-mon[23723]: pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:46.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:46 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/714120955' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:47.562 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:47.562 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:48.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:48 smithi042 ceph-mon[23723]: pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:49.454 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:49.454 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:49.456 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:50.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:50 smithi042 ceph-mon[23723]: pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:50.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:50 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/4214441623' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:51.036 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:51.036 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:52.735 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:52.735 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:52.737 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:52.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:52 smithi042 ceph-mon[23723]: pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:53.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:53 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1379400742' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:54.385 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:54.385 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:54.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:54 smithi042 ceph-mon[23723]: pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:56.207 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:56.207 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:56.209 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:47:56.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:56 smithi042 ceph-mon[23723]: pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:56 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2144127171' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:47:57.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:57 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:47:57.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:57 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:47:57.796 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:47:57.796 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:47:58.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:47:58 smithi042 ceph-mon[23723]: pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:47:59.637 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:47:59.637 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:47:59.639 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:00 smithi042 ceph-mon[23723]: pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:00 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3309714073' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:01.252 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:01.253 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:02.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:02 smithi042 ceph-mon[23723]: pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:02.944 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:02.945 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:02.947 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:03.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:03 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3882732681' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:04.516 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:04.516 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:04.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:04 smithi042 ceph-mon[23723]: pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:06.468 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:06.468 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:06.469 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:06.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:06 smithi042 ceph-mon[23723]: pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:07.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:07 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3701858094' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:08.067 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:08.067 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:08.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:08 smithi042 ceph-mon[23723]: pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:09.943 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:09.944 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:09.946 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:10 smithi042 ceph-mon[23723]: pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:10 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2780168405' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:11.568 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:11.569 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:12.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:12 smithi042 ceph-mon[23723]: pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:13.358 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:13.358 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:13.360 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:13.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:13 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3623711604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:14.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:14 smithi042 ceph-mon[23723]: pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:14.916 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:14.916 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:16.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:16 smithi042 ceph-mon[23723]: pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:16.857 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:16.857 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:16.859 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:17.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:17 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/280771149' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:18.427 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:18.428 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:18.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:18 smithi042 ceph-mon[23723]: pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:20.311 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:20.311 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:20.313 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:20.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:20 smithi042 ceph-mon[23723]: pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:20.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:20 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3539043246' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:21.897 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:21.898 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:22.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:22 smithi042 ceph-mon[23723]: pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:23.840 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:23.841 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:23.842 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:24.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:24 smithi042 ceph-mon[23723]: pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:24.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:24 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/862231902' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:25.392 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:25.393 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:26.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:26 smithi042 ceph-mon[23723]: pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:27.261 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:27.261 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:27.263 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:27.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:27 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/202554699' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:28.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:28 smithi042 ceph-mon[23723]: pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:28.839 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:28.839 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:30.651 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:30.652 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:30.654 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:30.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:30 smithi042 ceph-mon[23723]: pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:31.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:31 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3056683750' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:32.197 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:32.197 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:32.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:32 smithi042 ceph-mon[23723]: pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:33.952 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:33.953 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:33.954 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:34.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:34 smithi042 ceph-mon[23723]: pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:34.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:34 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/372708096' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:35.525 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:35.525 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:36.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:36 smithi042 ceph-mon[23723]: pgmap v73: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:37.379 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:37.379 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:37.380 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:37.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:37 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2065566920' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:38.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:38 smithi042 ceph-mon[23723]: pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:38.954 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:38.954 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:40.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:40 smithi042 ceph-mon[23723]: pgmap v75: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:40.835 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:40.835 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:40.837 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:41.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:41 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1991684783' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:42.416 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:42.417 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:42.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:42 smithi042 ceph-mon[23723]: pgmap v76: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:44.300 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:44.301 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:44.303 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:44.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:44 smithi042 ceph-mon[23723]: pgmap v77: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:44.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:44 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3295985032' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:45.874 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:45.875 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:46.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:46 smithi042 ceph-mon[23723]: pgmap v78: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:47.766 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:47.767 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:47.768 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:48.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:48 smithi042 ceph-mon[23723]: pgmap v79: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:48.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:48 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1538850311' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:49.313 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:49.313 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:50.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:50 smithi042 ceph-mon[23723]: pgmap v80: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:51.305 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:51.305 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:51.307 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:51.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:51 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2661544827' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:52.889 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:52.890 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:52.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:52 smithi042 ceph-mon[23723]: pgmap v81: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:54.693 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:54.694 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:54.696 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:54.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:54 smithi042 ceph-mon[23723]: pgmap v82: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:55.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:55 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1598919567' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:56.430 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:56.430 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:48:56.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:56 smithi042 ceph-mon[23723]: pgmap v83: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:57.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:57 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:48:57.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:57 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:48:58.285 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:48:58.285 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:48:58.286 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:48:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:58 smithi042 ceph-mon[23723]: pgmap v84: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:48:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:48:58 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3822690739' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:48:59.829 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:48:59.829 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:00.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:00 smithi042 ceph-mon[23723]: pgmap v85: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:01.731 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:01.731 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:01.733 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:02.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:02 smithi042 ceph-mon[23723]: pgmap v86: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:02 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2461342081' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:03.319 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:03.319 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:04.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:04 smithi042 ceph-mon[23723]: pgmap v87: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:05.258 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:05.258 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:05.260 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:05.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:05 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/200138937' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:06.843 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:06.844 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:06.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:06 smithi042 ceph-mon[23723]: pgmap v88: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:08.685 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:08.685 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:08.687 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:08.794 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:08 smithi042 ceph-mon[23723]: pgmap v89: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:09.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:09 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1838870901' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:10.264 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:10.264 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:10.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:10 smithi042 ceph-mon[23723]: pgmap v90: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:12.065 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:12.065 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:12.067 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:12.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:12 smithi042 ceph-mon[23723]: pgmap v91: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:12.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:12 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3231019288' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:13.676 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:13.676 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:14.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:14 smithi042 ceph-mon[23723]: pgmap v92: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:15.605 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:15.606 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:15.607 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:16.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:16 smithi042 ceph-mon[23723]: pgmap v93: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:16.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:16 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/273080801' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:17.180 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:17.180 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:18.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:18 smithi042 ceph-mon[23723]: pgmap v94: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:19.069 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:19.070 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:19.071 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:19.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:19 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/1420097536' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:20.638 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:20.638 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:20.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:20 smithi042 ceph-mon[23723]: pgmap v95: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:22.521 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:22.521 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:22.523 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:22.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:22 smithi042 ceph-mon[23723]: pgmap v96: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:22.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:22 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3482857163' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:24.094 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:24.094 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:24.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:24 smithi042 ceph-mon[23723]: pgmap v97: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:24.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:24 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:25.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:25 smithi042 ceph-mon[23723]: Deploying daemon node-exporter.smithi042 on smithi042 2024-09-13T12:49:26.088 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:26.088 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:26.091 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:26.780 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:26 smithi042 ceph-mon[23723]: pgmap v98: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:26.780 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:26 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3305220791' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:27.684 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:27.684 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:28.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:28 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:28.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:28 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T12:49:28.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:28 smithi042 ceph-mon[23723]: pgmap v99: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:28.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:28 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:29.406 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:29 smithi042 ceph-mon[23723]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T12:49:29.406 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:29 smithi042 ceph-mon[23723]: mgrmap e14: smithi042.jhsfyp(active, since 3m) 2024-09-13T12:49:29.566 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:29.566 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:29.568 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:30.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:30 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2438778818' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:31.147 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:31.147 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:32.990 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:32.991 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:32.993 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:33.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:33 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2031549690' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:34.549 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:34.549 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:35.669 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: Active manager daemon smithi042.jhsfyp restarted 2024-09-13T12:49:35.669 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: Activating manager daemon smithi042.jhsfyp 2024-09-13T12:49:35.669 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: mgrmap e15: smithi042.jhsfyp(active, starting, since 0.874235s) 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:35.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:49:35.671 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:35.671 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:49:35.671 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:49:36.606 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:36.607 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:36.608 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:37.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:37.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:36 smithi042 ceph-mon[23723]: mgrmap e16: smithi042.jhsfyp(active, since 1.87583s) 2024-09-13T12:49:37.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:37.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:36 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/4112853189' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:38.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:38 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:38.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:38 smithi042 ceph-mon[23723]: mgrmap e17: smithi042.jhsfyp(active, since 2s) 2024-09-13T12:49:38.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:38 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:38.459 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:38.460 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:39.469 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:39 smithi042 ceph-mon[23723]: [13/Sep/2024:12:49:37] ENGINE Bus STARTING 2024-09-13T12:49:39.469 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:39 smithi042 ceph-mon[23723]: [13/Sep/2024:12:49:37] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T12:49:39.469 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:39 smithi042 ceph-mon[23723]: [13/Sep/2024:12:49:37] ENGINE Bus STARTED 2024-09-13T12:49:40.693 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:40.694 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:40.696 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:41.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:40 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2660712410' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:41.987 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:41 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:41.987 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:41 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:41.987 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:41 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi046", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:49:42.277 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:42.277 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi042", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T12:49:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:44.276 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:44.276 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:44.278 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:44.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:43 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:49:44.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:43 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:49:44.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:43 smithi042 ceph-mon[23723]: Deploying daemon crash.smithi046 on smithi046 2024-09-13T12:49:45.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:44 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2393869184' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:46.320 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:46.320 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:48.519 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:48.519 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:48.521 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:49.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:49:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T12:49:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:49:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: Deploying daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T12:49:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:49 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2527969358' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:50.111 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:50.112 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:51.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:52.175 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:52.176 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:52.177 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:52.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:52 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3313482152' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:52.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:52.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T12:49:52.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:53.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:53 smithi042 ceph-mon[23723]: Deploying daemon mon.smithi046 on smithi046 2024-09-13T12:49:54.064 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:54.065 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:49:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:56 smithi042 ceph-mon[23723]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:49:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:56.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:56 smithi046 podman[32210]: 2024-09-13 12:49:56.810903152 +0000 UTC m=+0.297212915 container create 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, name=centos-stream, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, version=8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , RELEASE=HEAD, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, vcs-type=git) 2024-09-13T12:49:56.956 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:49:56.956 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:44:55.874236Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T12:49:56.958 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-09-13T12:49:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 podman[32210]: 2024-09-13 12:49:57.021990432 +0000 UTC m=+0.508300204 container init 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, vcs-type=git, maintainer=Guillaume Abrioux , release=754, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, name=centos-stream, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, architecture=x86_64, GIT_CLEAN=True) 2024-09-13T12:49:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 podman[32210]: 2024-09-13 12:49:57.026632002 +0000 UTC m=+0.512941768 container start 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, ceph=True, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, distribution-scope=public, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True) 2024-09-13T12:49:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T12:49:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pidfile_write: ignore empty --pid-file 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: load: jerasure load: lrc 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: RocksDB version: 6.15.5 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Compile date Apr 18 2022 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: DB SUMMARY 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: DB Session ID: 0L4WCNM6SWNCUSKS707M 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: CURRENT file: CURRENT 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi046/store.db dir, Total Num: 0, files: 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi046/store.db: 000004.log size: 511 ; 2024-09-13T12:49:57.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.error_if_exists: 0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.create_if_missing: 0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.paranoid_checks: 1 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.env: 0x55f724ac6860 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.fs: Posix File System 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.info_log: 0x55f7272c5fa0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.statistics: (nil) 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.use_fsync: 0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_log_file_size: 0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T12:49:57.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.allow_fallocate: 1 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.use_direct_reads: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.db_log_dir: 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi046/store.db 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T12:49:57.370 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T12:49:57.371 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T12:49:57.372 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.write_buffer_manager: 0x55f7273b6330 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.unordered_write: 0 2024-09-13T12:49:57.373 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.row_cache: None 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.wal_filter: None 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.preserve_deletes: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.two_write_queues: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.atomic_flush: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T12:49:57.374 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.log_readahead_size: 0 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_background_jobs: 2 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_background_compactions: -1 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_subcompactions: 1 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T12:49:57.375 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_open_files: -1 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_background_flushes: -1 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Compression algorithms supported: 2024-09-13T12:49:57.376 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kZSTD supported: 0 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kXpressCompression supported: 0 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kLZ4Compression supported: 1 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kBZip2Compression supported: 0 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kZlibCompression supported: 1 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: kSnappyCompression supported: 1 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi046/store.db/MANIFEST-000003 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.merge_operator: 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_filter: None 2024-09-13T12:49:57.377 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f727291d00) 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: cache_index_and_filter_blocks: 1 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: pin_top_level_index_and_filter: 1 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: index_type: 0 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: data_block_index_type: 0 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: index_shortening: 1 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: hash_index_allow_collision: 1 2024-09-13T12:49:57.378 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: checksum: 1 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: no_block_cache: 0 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache: 0x55f7272fc170 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache_name: BinnedLRUCache 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache_options: 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: capacity : 536870912 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: num_shard_bits : 4 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: strict_capacity_limit : 0 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: high_pri_pool_ratio: 0.000 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache_compressed: (nil) 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: persistent_cache: (nil) 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_size: 4096 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_size_deviation: 10 2024-09-13T12:49:57.379 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_restart_interval: 16 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: index_block_restart_interval: 1 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: metadata_block_size: 4096 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: partition_filters: 0 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: use_delta_encoding: 1 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: whole_key_filtering: 1 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: verify_compression: 0 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: read_amp_bytes_per_bit: 0 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: format_version: 4 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: enable_index_compression: 1 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_align: 0 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T12:49:57.380 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression: NoCompression 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.num_levels: 7 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T12:49:57.381 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T12:49:57.382 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.arena_block_size: 4194304 2024-09-13T12:49:57.383 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T12:49:57.384 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.table_properties_collectors: 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.inplace_update_support: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.bloom_locality: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.max_successive_merges: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T12:49:57.385 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.ttl: 2592000 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.enable_blob_files: false 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.min_blob_size: 0 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi046/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726231797053884, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T12:49:57.386 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726231797054193, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726231797, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "41aa36f7-db67-4ed3-b936-1be50916fad8", "db_session_id": "0L4WCNM6SWNCUSKS707M"}} 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726231797054514, "job": 1, "event": "recovery_finished"} 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi046/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55f7272e2a80 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: DB pointer 0x55f7272f2000 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** DB Stats ** 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T12:49:57.387 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** Compaction Stats [default] ** 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** Compaction Stats [default] ** 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:49:57.388 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval compaction: 0.00 GB write, 0.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T12:49:57.389 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** Compaction Stats [default] ** 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** Compaction Stats [default] ** 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T12:49:57.390 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative compaction: 0.00 GB write, 0.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046 does not exist in monmap, will attempt to join an existing cluster 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: using public_addr v2:172.21.15.46:0/0 -> [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: starting mon.smithi046 rank -1 at public addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] at bind addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi046 fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(???) e0 preinit fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:49:57.391 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).mds e1 new map 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).mds e1 print_map 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: e1 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: legacy client fscid: -1 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: No filesystems configured 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T12:49:57.392 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/1598919567' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v83: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v84: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3822690739' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v85: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v86: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.393 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2461342081' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v87: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/200138937' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v88: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v89: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/1838870901' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v90: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v91: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3231019288' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v92: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v93: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/273080801' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v94: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/1420097536' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.394 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v95: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v96: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3482857163' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v97: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Deploying daemon node-exporter.smithi042 on smithi042 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v98: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3305220791' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v99: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14162 172.21.15.42:0/2006102446' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T12:49:57.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mgrmap e14: smithi042.jhsfyp(active, since 3m) 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2438778818' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2031549690' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Active manager daemon smithi042.jhsfyp restarted 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Activating manager daemon smithi042.jhsfyp 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mgrmap e15: smithi042.jhsfyp(active, starting, since 0.874235s) 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:49:57.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mgrmap e16: smithi042.jhsfyp(active, since 1.87583s) 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/4112853189' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mgrmap e17: smithi042.jhsfyp(active, since 2s) 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: [13/Sep/2024:12:49:37] ENGINE Bus STARTING 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: [13/Sep/2024:12:49:37] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: [13/Sep/2024:12:49:37] ENGINE Bus STARTED 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2660712410' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi046", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd/host:smithi042", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Deploying daemon crash.smithi046 on smithi046 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2393869184' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Deploying daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T12:49:57.399 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2527969358' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3313482152' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: Deploying daemon mon.smithi046 on smithi046 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-09-13T12:49:57.400 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 ceph-mon[32268]: mon.smithi046@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-09-13T12:49:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 bash[32210]: 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 2024-09-13T12:49:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:49:57 smithi046 systemd[1]: Started Ceph mon.smithi046 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T12:49:58.767 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-09-13T12:49:58.768 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T12:50:02.395 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: mon.smithi042 calling monitor election 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: mon.smithi046 calling monitor election 2024-09-13T12:50:02.396 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: mon.smithi042 is new leader, mons smithi042,smithi046 in quorum (ranks 0,1) 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: monmap e2: 2 mons at {smithi042=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],smithi046=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0]} 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: fsmap 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: mgrmap e17: smithi042.jhsfyp(active, since 27s) 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm started 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:50:02.397 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: overall HEALTH_OK 2024-09-13T12:50:02.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T12:50:02.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:50:02.398 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:02 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:02.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:50:02.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: mon.smithi042 calling monitor election 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: mon.smithi046 calling monitor election 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: mon.smithi042 is new leader, mons smithi042,smithi046 in quorum (ranks 0,1) 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: monmap e2: 2 mons at {smithi042=[v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0],smithi046=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0]} 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: fsmap 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: osdmap e5: 0 total, 0 up, 0 in 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: mgrmap e17: smithi042.jhsfyp(active, since 27s) 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: Standby manager daemon smithi046.hcgddm started 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: overall HEALTH_OK 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/4292369465' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:50:02.494 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:02 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:03.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:03 smithi042 ceph-mon[23723]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:03.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:03 smithi042 ceph-mon[23723]: Deploying daemon node-exporter.smithi046 on smithi046 2024-09-13T12:50:03.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:03 smithi042 ceph-mon[23723]: mgrmap e18: smithi042.jhsfyp(active, since 27s), standbys: smithi046.hcgddm 2024-09-13T12:50:03.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:03 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:50:03.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:03 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:03.615 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:03 smithi046 ceph-mon[32268]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:03.615 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:03 smithi046 ceph-mon[32268]: Deploying daemon node-exporter.smithi046 on smithi046 2024-09-13T12:50:03.615 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:03 smithi046 ceph-mon[32268]: mgrmap e18: smithi042.jhsfyp(active, since 27s), standbys: smithi046.hcgddm 2024-09-13T12:50:03.615 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:03 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:50:03.615 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:03 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:50:03.615 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:50:03.615 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":2,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","modified":"2024-09-13T12:49:57.082008Z","created":"2024-09-13T12:44:55.874236Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi042","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:3300","nonce":0},{"type":"v1","addr":"172.21.15.42:6789","nonce":0}]},"addr":"172.21.15.42:6789/0","public_addr":"172.21.15.42:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi046","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:3300","nonce":0},{"type":"v1","addr":"172.21.15.46:6789","nonce":0}]},"addr":"172.21.15.46:6789/0","public_addr":"172.21.15.46:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-13T12:50:03.616 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 2 2024-09-13T12:50:04.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:04 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3523597222' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:50:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:04 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3523597222' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T12:50:05.318 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:05 smithi046 ceph-mon[32268]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:05.339 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T12:50:05.339 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T12:50:05.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:05 smithi042 ceph-mon[23723]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:06.693 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:06 smithi042 ceph-mon[23723]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:06.693 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:06 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:06.693 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:06 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:06.693 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:06 smithi042 ceph-mon[23723]: Deploying daemon prometheus.smithi042 on smithi042 2024-09-13T12:50:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:06 smithi046 ceph-mon[32268]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:06 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:06 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:06 smithi046 ceph-mon[32268]: Deploying daemon prometheus.smithi042 on smithi042 2024-09-13T12:50:07.367 INFO:teuthology.orchestra.run.smithi042.stdout:# minimal ceph.conf for e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:50:07.367 INFO:teuthology.orchestra.run.smithi042.stdout:[global] 2024-09-13T12:50:07.367 INFO:teuthology.orchestra.run.smithi042.stdout: fsid = e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T12:50:07.367 INFO:teuthology.orchestra.run.smithi042.stdout: mon_host = [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] 2024-09-13T12:50:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:08 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/542591098' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:08.413 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:08 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/542591098' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:08.446 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T12:50:08.446 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:50:08.446 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T12:50:08.480 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:50:08.480 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:50:08.549 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-09-13T12:50:08.550 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T12:50:08.584 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-09-13T12:50:08.584 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:50:08.654 DEBUG:tasks.cephadm:set 0 configs 2024-09-13T12:50:08.654 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T12:50:08.654 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:50:08.654 DEBUG:teuthology.orchestra.run.smithi042:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T12:50:08.670 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T12:50:08.670 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_1 2024-09-13T12:50:08.727 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T12:50:08.727 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:08.727 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-09-13T12:50:08.727 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:08.727 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:08.728 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 12:49:24.701986535 +0000 2024-09-13T12:50:08.728 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 12:44:03.843182910 +0000 2024-09-13T12:50:08.728 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 12:44:03.843182910 +0000 2024-09-13T12:50:08.728 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 12:44:03.843182910 +0000 2024-09-13T12:50:08.728 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T12:50:08.792 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T12:50:08.792 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T12:50:08.792 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000109466 s, 4.7 MB/s 2024-09-13T12:50:08.793 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T12:50:08.851 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_2 2024-09-13T12:50:08.906 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 12:49:24.702986527 +0000 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 12:44:04.155180372 +0000 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 12:44:04.155180372 +0000 2024-09-13T12:50:08.907 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 12:44:04.155180372 +0000 2024-09-13T12:50:08.907 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T12:50:08.972 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T12:50:08.973 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T12:50:08.973 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 9.2273e-05 s, 5.5 MB/s 2024-09-13T12:50:08.974 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T12:50:09.032 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_3 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 706 Links: 1 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 12:49:24.702986527 +0000 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 12:44:04.468177825 +0000 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 12:44:04.468177825 +0000 2024-09-13T12:50:09.089 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 12:44:04.468177825 +0000 2024-09-13T12:50:09.090 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T12:50:09.152 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T12:50:09.153 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T12:50:09.153 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.00010759 s, 4.8 MB/s 2024-09-13T12:50:09.154 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T12:50:09.211 DEBUG:teuthology.orchestra.run.smithi042:> stat /dev/vg_nvme/lv_4 2024-09-13T12:50:09.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:09 smithi042 ceph-mon[23723]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:09.256 INFO:teuthology.orchestra.run.smithi042.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout:Access: 2024-09-13 12:49:24.702986527 +0000 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout:Modify: 2024-09-13 12:44:04.796175157 +0000 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout:Change: 2024-09-13 12:44:04.796175157 +0000 2024-09-13T12:50:09.257 INFO:teuthology.orchestra.run.smithi042.stdout: Birth: 2024-09-13 12:44:04.796175157 +0000 2024-09-13T12:50:09.257 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T12:50:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records in 2024-09-13T12:50:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:1+0 records out 2024-09-13T12:50:09.321 INFO:teuthology.orchestra.run.smithi042.stderr:512 bytes copied, 0.000105519 s, 4.9 MB/s 2024-09-13T12:50:09.322 DEBUG:teuthology.orchestra.run.smithi042:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T12:50:09.366 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:09 smithi046 ceph-mon[32268]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:09.379 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-09-13T12:50:09.379 DEBUG:teuthology.orchestra.run.smithi046:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T12:50:09.399 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T12:50:09.399 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_1 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-09-13 12:50:02.689162829 +0000 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-09-13 12:44:03.120920590 +0000 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-09-13 12:44:03.120920590 +0000 2024-09-13T12:50:09.459 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-09-13 12:44:03.120920590 +0000 2024-09-13T12:50:09.460 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T12:50:09.528 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-09-13T12:50:09.529 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-09-13T12:50:09.529 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.00013187 s, 3.9 MB/s 2024-09-13T12:50:09.530 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T12:50:09.589 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_2 2024-09-13T12:50:09.647 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T12:50:09.647 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-09-13 12:50:02.689162829 +0000 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-09-13 12:44:03.448916891 +0000 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-09-13 12:44:03.448916891 +0000 2024-09-13T12:50:09.648 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-09-13 12:44:03.448916891 +0000 2024-09-13T12:50:09.648 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T12:50:09.717 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-09-13T12:50:09.717 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-09-13T12:50:09.717 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.000109365 s, 4.7 MB/s 2024-09-13T12:50:09.717 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T12:50:09.776 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_3 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-09-13 12:50:02.689162829 +0000 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-09-13 12:44:03.769913271 +0000 2024-09-13T12:50:09.834 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-09-13 12:44:03.769913271 +0000 2024-09-13T12:50:09.835 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-09-13 12:44:03.769913271 +0000 2024-09-13T12:50:09.835 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T12:50:09.900 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-09-13T12:50:09.900 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-09-13T12:50:09.904 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.000119482 s, 4.3 MB/s 2024-09-13T12:50:09.905 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T12:50:09.962 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_4 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-09-13 12:50:02.690162818 +0000 2024-09-13T12:50:10.019 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-09-13 12:44:04.104909494 +0000 2024-09-13T12:50:10.020 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-09-13 12:44:04.104909494 +0000 2024-09-13T12:50:10.020 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-09-13 12:44:04.104909494 +0000 2024-09-13T12:50:10.020 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T12:50:10.085 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-09-13T12:50:10.085 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-09-13T12:50:10.085 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.000114 s, 4.5 MB/s 2024-09-13T12:50:10.086 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T12:50:10.144 INFO:tasks.cephadm:Deploying osd.0 on smithi042 with /dev/vg_nvme/lv_4... 2024-09-13T12:50:10.144 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T12:50:10.459 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:10 smithi042 ceph-mon[23723]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:10 smithi046 ceph-mon[32268]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:11.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:11 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:11.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:11 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:12.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:12 smithi042 ceph-mon[23723]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:12.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:12 smithi046 ceph-mon[32268]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:14.719 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:14 smithi042 ceph-mon[23723]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:14 smithi046 ceph-mon[32268]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:16.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:16 smithi042 ceph-mon[23723]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:16 smithi046 ceph-mon[32268]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:18 smithi046 ceph-mon[32268]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:18 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:18 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:50:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:18 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:18 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:50:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:18 smithi042 ceph-mon[23723]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:18 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:18 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:50:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:18 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:18 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:50:20.261 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:50:20.282 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_4 2024-09-13T12:50:21.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:21 smithi042 ceph-mon[23723]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:21.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:21.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:21 smithi042 ceph-mon[23723]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T12:50:21.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:21 smithi042 ceph-mon[23723]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:50:21.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:21.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:21 smithi046 ceph-mon[32268]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:21 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T12:50:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:21 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:50:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:23.235 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:23 smithi042 ceph-mon[23723]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:23.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:23 smithi046 ceph-mon[32268]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T12:50:25.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T12:50:25.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T12:50:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T12:50:25.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:26.128 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:26 smithi042 ceph-mon[23723]: from='client.14336 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:50:26.128 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:26 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:50:26.128 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:26 smithi042 ceph-mon[23723]: Reconfiguring mon.smithi042 (unknown last config time)... 2024-09-13T12:50:26.129 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:26 smithi042 ceph-mon[23723]: Reconfiguring daemon mon.smithi042 on smithi042 2024-09-13T12:50:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:26 smithi046 ceph-mon[32268]: from='client.14336 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:50:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:26 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:50:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:26 smithi046 ceph-mon[32268]: Reconfiguring mon.smithi042 (unknown last config time)... 2024-09-13T12:50:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:26 smithi046 ceph-mon[32268]: Reconfiguring daemon mon.smithi042 on smithi042 2024-09-13T12:50:27.115 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:27 smithi042 ceph-mon[23723]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:27.116 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:27 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:27.116 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:27 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:50:27.116 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:27 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:50:27.116 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:27 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:27 smithi046 ceph-mon[32268]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:27 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:27 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:50:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:27 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:50:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:27 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:28 smithi046 ceph-mon[32268]: Reconfiguring mgr.smithi042.jhsfyp (unknown last config time)... 2024-09-13T12:50:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:28 smithi046 ceph-mon[32268]: Reconfiguring daemon mgr.smithi042.jhsfyp on smithi042 2024-09-13T12:50:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:28 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2403556090' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edec8271-f2f4-4f2b-b7b8-710f44c128fa"}]: dispatch 2024-09-13T12:50:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:28 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2403556090' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "edec8271-f2f4-4f2b-b7b8-710f44c128fa"}]': finished 2024-09-13T12:50:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:28 smithi046 ceph-mon[32268]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T12:50:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:28 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:28.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:28 smithi042 ceph-mon[23723]: Reconfiguring mgr.smithi042.jhsfyp (unknown last config time)... 2024-09-13T12:50:28.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:28 smithi042 ceph-mon[23723]: Reconfiguring daemon mgr.smithi042.jhsfyp on smithi042 2024-09-13T12:50:28.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:28 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2403556090' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "edec8271-f2f4-4f2b-b7b8-710f44c128fa"}]: dispatch 2024-09-13T12:50:28.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:28 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2403556090' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "edec8271-f2f4-4f2b-b7b8-710f44c128fa"}]': finished 2024-09-13T12:50:28.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:28 smithi042 ceph-mon[23723]: osdmap e6: 1 total, 0 up, 1 in 2024-09-13T12:50:28.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:28 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:29.366 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:29 smithi046 ceph-mon[32268]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:29.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:29 smithi042 ceph-mon[23723]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:30 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3381643416' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:50:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:30 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3381643416' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:50:31.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:31 smithi046 ceph-mon[32268]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:31.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:31 smithi042 ceph-mon[23723]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:33.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:33 smithi042 ceph-mon[23723]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:33.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:33 smithi046 ceph-mon[32268]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:34 smithi042 ceph-mon[23723]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:34 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:34 smithi042 ceph-mon[23723]: Reconfiguring alertmanager.smithi042 (dependencies changed)... 2024-09-13T12:50:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:34 smithi042 ceph-mon[23723]: Reconfiguring daemon alertmanager.smithi042 on smithi042 2024-09-13T12:50:34.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:34 smithi046 ceph-mon[32268]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:34 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:34 smithi046 ceph-mon[32268]: Reconfiguring alertmanager.smithi042 (dependencies changed)... 2024-09-13T12:50:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:34 smithi046 ceph-mon[32268]: Reconfiguring daemon alertmanager.smithi042 on smithi042 2024-09-13T12:50:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:36 smithi042 ceph-mon[23723]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:50:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:50:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:36 smithi046 ceph-mon[32268]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:50:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:50:38.683 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:38 smithi042 ceph-mon[23723]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:38.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:38 smithi046 ceph-mon[32268]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:40.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:40.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:39 smithi042 ceph-mon[23723]: Reconfiguring crash.smithi042 (monmap changed)... 2024-09-13T12:50:40.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:50:40.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:40.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:39 smithi042 ceph-mon[23723]: Reconfiguring daemon crash.smithi042 on smithi042 2024-09-13T12:50:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:39 smithi046 ceph-mon[32268]: Reconfiguring crash.smithi042 (monmap changed)... 2024-09-13T12:50:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:50:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:39 smithi046 ceph-mon[32268]: Reconfiguring daemon crash.smithi042 on smithi042 2024-09-13T12:50:41.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:40 smithi042 ceph-mon[23723]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:41.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:40 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T12:50:41.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:40 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:41.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:40 smithi046 ceph-mon[32268]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:41.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:40 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T12:50:41.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:40 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:41.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:41 smithi042 ceph-mon[23723]: Deploying daemon osd.0 on smithi042 2024-09-13T12:50:41.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:41 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:42.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:41 smithi046 ceph-mon[32268]: Deploying daemon osd.0 on smithi042 2024-09-13T12:50:42.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:41 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:43.215 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:42 smithi042 ceph-mon[23723]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:43.215 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:42 smithi042 ceph-mon[23723]: Reconfiguring grafana.smithi042 (dependencies changed)... 2024-09-13T12:50:43.215 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:42 smithi042 ceph-mon[23723]: Reconfiguring daemon grafana.smithi042 on smithi042 2024-09-13T12:50:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:42 smithi046 ceph-mon[32268]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:42 smithi046 ceph-mon[32268]: Reconfiguring grafana.smithi042 (dependencies changed)... 2024-09-13T12:50:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:42 smithi046 ceph-mon[32268]: Reconfiguring daemon grafana.smithi042 on smithi042 2024-09-13T12:50:45.216 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:44 smithi042 ceph-mon[23723]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:44 smithi046 ceph-mon[32268]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:47.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:46 smithi042 ceph-mon[23723]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:46 smithi046 ceph-mon[32268]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:48 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:48.386 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:48 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:49.366 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:49 smithi046 ceph-mon[32268]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:49.408 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:49 smithi042 ceph-mon[23723]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:50.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:50 smithi042 ceph-mon[23723]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:50.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:50 smithi042 ceph-mon[23723]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T12:50:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:50 smithi046 ceph-mon[32268]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:50 smithi046 ceph-mon[32268]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T12:50:51.630 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:51 smithi042 ceph-mon[23723]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T12:50:51.630 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:51 smithi042 ceph-mon[23723]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T12:50:51.630 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:51.630 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:51 smithi042 ceph-mon[23723]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:50:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:51 smithi046 ceph-mon[32268]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T12:50:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:51 smithi046 ceph-mon[32268]: osdmap e7: 1 total, 0 up, 1 in 2024-09-13T12:50:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:51 smithi046 ceph-mon[32268]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:50:52.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:52.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:50:52.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T12:50:52.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:52.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:52.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:52.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: Reconfiguring crash.smithi046 (monmap changed)... 2024-09-13T12:50:52.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:50:52.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:52.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:52 smithi042 ceph-mon[23723]: Reconfiguring daemon crash.smithi046 on smithi046 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: from='osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: osdmap e8: 1 total, 0 up, 1 in 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: Reconfiguring crash.smithi046 (monmap changed)... 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-09-13T12:50:52.695 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:52.696 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:52 smithi046 ceph-mon[32268]: Reconfiguring daemon crash.smithi046 on smithi046 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529] boot 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:50:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: osd.0 [v2:172.21.15.42:6802/3230857529,v1:172.21.15.42:6803/3230857529] boot 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: osdmap e9: 1 total, 1 up, 1 in 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:50:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:53.677 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 0 on host 'smithi042' 2024-09-13T12:50:54.291 DEBUG:teuthology.orchestra.run.smithi042:osd.0> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T12:50:54.294 INFO:tasks.cephadm:Deploying osd.1 on smithi042 with /dev/vg_nvme/lv_3... 2024-09-13T12:50:54.294 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T12:50:54.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:54 smithi042 ceph-mon[23723]: Reconfiguring mgr.smithi046.hcgddm (monmap changed)... 2024-09-13T12:50:54.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:54 smithi042 ceph-mon[23723]: Reconfiguring daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T12:50:54.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:54 smithi042 ceph-mon[23723]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:54.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:54 smithi042 ceph-mon[23723]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T12:50:54.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:54.712 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:54 smithi046 ceph-mon[32268]: Reconfiguring mgr.smithi046.hcgddm (monmap changed)... 2024-09-13T12:50:54.713 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:54 smithi046 ceph-mon[32268]: Reconfiguring daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T12:50:54.713 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:54 smithi046 ceph-mon[32268]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T12:50:54.713 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:54 smithi046 ceph-mon[32268]: osdmap e10: 1 total, 1 up, 1 in 2024-09-13T12:50:54.713 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:55 smithi042 ceph-mon[23723]: Reconfiguring mon.smithi046 (monmap changed)... 2024-09-13T12:50:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T12:50:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T12:50:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:55 smithi042 ceph-mon[23723]: Reconfiguring daemon mon.smithi046 on smithi046 2024-09-13T12:50:55.996 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:55.996 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:55 smithi046 ceph-mon[32268]: Reconfiguring mon.smithi046 (monmap changed)... 2024-09-13T12:50:55.996 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T12:50:55.996 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T12:50:55.997 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:55.997 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:55 smithi046 ceph-mon[32268]: Reconfiguring daemon mon.smithi046 on smithi046 2024-09-13T12:50:56.915 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:50:56.940 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_3 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-09-13T12:50:57.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.42:3000"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.42:3000"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.42:9095"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.42:9095"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:57.243 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: pgmap v38: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.42:9093"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.42:3000"}]: dispatch 2024-09-13T12:50:57.332 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.42:3000"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.42:9095"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.42:9095"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:50:57.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:50:58.939 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:58 smithi042 ceph-mon[23723]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:50:58.939 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:58 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:58 smithi046 ceph-mon[32268]: pgmap v39: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:50:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:58 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:50:59.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:50:59.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:50:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:50:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:51:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:51:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:50:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:00.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:00 smithi042 ceph-mon[23723]: from='client.14354 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:51:00.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:00 smithi042 ceph-mon[23723]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:00 smithi046 ceph-mon[32268]: from='client.14354 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:51:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:00 smithi046 ceph-mon[32268]: pgmap v40: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:02.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:02 smithi042 ceph-mon[23723]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:02 smithi046 ceph-mon[32268]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:03.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:03 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1405946858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a68de12b-5ed6-474f-9a88-08d119714e95"}]: dispatch 2024-09-13T12:51:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:03 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1405946858' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a68de12b-5ed6-474f-9a88-08d119714e95"}]': finished 2024-09-13T12:51:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:03 smithi042 ceph-mon[23723]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T12:51:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:03 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:03 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2968979661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:51:04.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:03 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1405946858' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a68de12b-5ed6-474f-9a88-08d119714e95"}]: dispatch 2024-09-13T12:51:04.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:03 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1405946858' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a68de12b-5ed6-474f-9a88-08d119714e95"}]': finished 2024-09-13T12:51:04.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:03 smithi046 ceph-mon[32268]: osdmap e11: 2 total, 1 up, 2 in 2024-09-13T12:51:04.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:03 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:04.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:03 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2968979661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:51:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:04 smithi046 ceph-mon[32268]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:04 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:05.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:04 smithi042 ceph-mon[23723]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:05.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:04 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:06.974 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:06 smithi042 ceph-mon[23723]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:06 smithi046 ceph-mon[32268]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:08 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:09.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:08 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:09.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:09 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi042 2024-09-13T12:51:09.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:09 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi042 to 12868M 2024-09-13T12:51:10.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:09 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi042 2024-09-13T12:51:10.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:09 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi042 to 12868M 2024-09-13T12:51:11.436 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:11 smithi042 ceph-mon[23723]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:11.437 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:11 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:11.437 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:11 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T12:51:11.437 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:11 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:11 smithi046 ceph-mon[32268]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:11 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:11 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T12:51:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:11 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:12.235 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:12 smithi042 ceph-mon[23723]: Deploying daemon osd.1 on smithi042 2024-09-13T12:51:12.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:12 smithi046 ceph-mon[32268]: Deploying daemon osd.1 on smithi042 2024-09-13T12:51:13.200 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:13 smithi042 ceph-mon[23723]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:13 smithi046 ceph-mon[32268]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:15.377 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:15 smithi042 ceph-mon[23723]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:15 smithi046 ceph-mon[32268]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:17.464 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:17 smithi042 ceph-mon[23723]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:17.464 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:17 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:17.464 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:17 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:17 smithi046 ceph-mon[32268]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:17 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:17 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:19 smithi046 ceph-mon[32268]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:19.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:19 smithi042 ceph-mon[23723]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:19.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:19.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:19.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:19.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:21.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:21 smithi042 ceph-mon[23723]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:21.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:21.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:21 smithi046 ceph-mon[32268]: pgmap v51: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:21.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:21.645 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 1 on host 'smithi042' 2024-09-13T12:51:22.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:22 smithi042 ceph-mon[23723]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T12:51:22.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:22.267 DEBUG:teuthology.orchestra.run.smithi042:osd.1> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T12:51:22.269 INFO:tasks.cephadm:Deploying osd.2 on smithi042 with /dev/vg_nvme/lv_2... 2024-09-13T12:51:22.269 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T12:51:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:22 smithi046 ceph-mon[32268]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T12:51:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-mon[23723]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-mon[23723]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T12:51:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-mon[23723]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T12:51:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-mon[23723]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:51:23.493 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[38296]: 2024-09-13T12:51:23.201+0000 7fb49ffd7700 -1 osd.1 0 waiting for initial osdmap 2024-09-13T12:51:23.493 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:23 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[38296]: 2024-09-13T12:51:23.204+0000 7fb49896a700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T12:51:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:23 smithi046 ceph-mon[32268]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:23 smithi046 ceph-mon[32268]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T12:51:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:23 smithi046 ceph-mon[32268]: osdmap e12: 2 total, 1 up, 2 in 2024-09-13T12:51:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:23 smithi046 ceph-mon[32268]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:51:24.405 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:24 smithi042 ceph-mon[23723]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:51:24.405 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:24 smithi042 ceph-mon[23723]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T12:51:24.405 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:24 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:24.405 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:24 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:24 smithi046 ceph-mon[32268]: from='osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:51:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:24 smithi046 ceph-mon[32268]: osdmap e13: 2 total, 1 up, 2 in 2024-09-13T12:51:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:24 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:24 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:25.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:51:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:51:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428] boot 2024-09-13T12:51:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T12:51:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: osd.1 [v2:172.21.15.42:6810/1160947428,v1:172.21.15.42:6811/1160947428] boot 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: osdmap e14: 2 total, 2 up, 2 in 2024-09-13T12:51:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:51:25.951 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:51:25.971 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_2 2024-09-13T12:51:26.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:26 smithi042 ceph-mon[23723]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:26.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:26 smithi042 ceph-mon[23723]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T12:51:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:26 smithi046 ceph-mon[32268]: pgmap v57: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:26 smithi046 ceph-mon[32268]: osdmap e15: 2 total, 2 up, 2 in 2024-09-13T12:51:29.317 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:29 smithi042 ceph-mon[23723]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:29.317 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:29 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:29 smithi046 ceph-mon[32268]: pgmap v59: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:29 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:30.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:51:30.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:51:30.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:30.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:51:30.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:51:30.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:31.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:31 smithi042 ceph-mon[23723]: from='client.14372 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:51:31.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:31 smithi042 ceph-mon[23723]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:31 smithi046 ceph-mon[32268]: from='client.14372 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:51:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:31 smithi046 ceph-mon[32268]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:33.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: pgmap v61: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:34 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi042 to 6434M 2024-09-13T12:51:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:34 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1209564958' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0fa5feee-949c-46a9-8edc-5bc91cbcbe0c"}]: dispatch 2024-09-13T12:51:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:34 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1209564958' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0fa5feee-949c-46a9-8edc-5bc91cbcbe0c"}]': finished 2024-09-13T12:51:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:34 smithi042 ceph-mon[23723]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T12:51:34.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:34 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:34.518 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:34 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi042 to 6434M 2024-09-13T12:51:34.519 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:34 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1209564958' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0fa5feee-949c-46a9-8edc-5bc91cbcbe0c"}]: dispatch 2024-09-13T12:51:34.519 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:34 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1209564958' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0fa5feee-949c-46a9-8edc-5bc91cbcbe0c"}]': finished 2024-09-13T12:51:34.519 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:34 smithi046 ceph-mon[32268]: osdmap e16: 3 total, 2 up, 3 in 2024-09-13T12:51:34.519 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:34 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:35.459 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:35 smithi042 ceph-mon[23723]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:35.460 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:35 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/161549769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:51:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:35 smithi046 ceph-mon[32268]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:35 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/161549769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:51:36.447 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:51:36.447 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:51:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:51:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:51:37.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:37 smithi042 ceph-mon[23723]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:37.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:37 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:37.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:37 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:37 smithi046 ceph-mon[32268]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:37 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:37 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:39.405 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:39 smithi042 ceph-mon[23723]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:39.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:39 smithi046 ceph-mon[32268]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:40.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:40 smithi042 ceph-mon[23723]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:40.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:40 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T12:51:40.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:40 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:40.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:40 smithi042 ceph-mon[23723]: Deploying daemon osd.2 on smithi042 2024-09-13T12:51:40.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:40 smithi046 ceph-mon[32268]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:40 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T12:51:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:40 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:40 smithi046 ceph-mon[32268]: Deploying daemon osd.2 on smithi042 2024-09-13T12:51:42.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:42 smithi042 ceph-mon[23723]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:42 smithi046 ceph-mon[32268]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:44.563 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:44 smithi042 ceph-mon[23723]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:44.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:44 smithi046 ceph-mon[32268]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:46.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:46 smithi042 ceph-mon[23723]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:46.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:46.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:46.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:46.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:46 smithi046 ceph-mon[32268]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:48.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:48 smithi042 ceph-mon[23723]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:48.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:48 smithi046 ceph-mon[32268]: pgmap v70: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:49.220 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 2 on host 'smithi042' 2024-09-13T12:51:49.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:49 smithi042 ceph-mon[23723]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T12:51:49.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:49 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:49.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:49 smithi046 ceph-mon[32268]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T12:51:49.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:49 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:50.503 DEBUG:teuthology.orchestra.run.smithi042:osd.2> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T12:51:50.506 INFO:tasks.cephadm:Deploying osd.3 on smithi042 with /dev/vg_nvme/lv_1... 2024-09-13T12:51:50.506 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T12:51:50.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:50 smithi042 ceph-mon[23723]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:50 smithi042 ceph-mon[23723]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T12:51:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:50 smithi042 ceph-mon[23723]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T12:51:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:50 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:50 smithi042 ceph-mon[23723]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:51:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:50 smithi046 ceph-mon[32268]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:50 smithi046 ceph-mon[32268]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T12:51:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:50 smithi046 ceph-mon[32268]: osdmap e17: 3 total, 2 up, 3 in 2024-09-13T12:51:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:50 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:50 smithi046 ceph-mon[32268]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:51:51.734 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:51 smithi042 ceph-mon[23723]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:51:51.734 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:51 smithi042 ceph-mon[23723]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T12:51:51.734 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:51.734 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:51 smithi046 ceph-mon[32268]: from='osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:51:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:51 smithi046 ceph-mon[32268]: osdmap e18: 3 total, 2 up, 3 in 2024-09-13T12:51:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:52.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:51:52.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:51:52.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: pgmap v74: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:52.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:52.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957] boot 2024-09-13T12:51:52.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T12:51:52.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:51:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:51:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: pgmap v74: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-09-13T12:51:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:52.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: osd.2 [v2:172.21.15.42:6818/912984957,v1:172.21.15.42:6819/912984957] boot 2024-09-13T12:51:52.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: osdmap e19: 3 total, 3 up, 3 in 2024-09-13T12:51:52.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:51:53.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:53 smithi042 ceph-mon[23723]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T12:51:54.253 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:51:54.271 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi042:vg_nvme/lv_1 2024-09-13T12:51:54.366 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:53 smithi046 ceph-mon[32268]: osdmap e20: 3 total, 3 up, 3 in 2024-09-13T12:51:55.061 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:54 smithi042 ceph-mon[23723]: pgmap v77: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:51:55.062 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T12:51:55.062 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:55.062 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T12:51:55.062 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:54 smithi042 ceph-mon[23723]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T12:51:55.062 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T12:51:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:54 smithi046 ceph-mon[32268]: pgmap v77: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:51:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T12:51:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T12:51:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:54 smithi046 ceph-mon[32268]: osdmap e21: 3 total, 3 up, 3 in 2024-09-13T12:51:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T12:51:55.948 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T12:51:55.948 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:55 smithi042 ceph-mon[23723]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T12:51:56.242 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 12:51:55 smithi042 sudo[43462]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T12:51:56.242 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 12:51:55 smithi042 sudo[43462]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:56.242 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 12:51:55 smithi042 sudo[43462]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:56.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T12:51:56.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:55 smithi046 ceph-mon[32268]: osdmap e22: 3 total, 3 up, 3 in 2024-09-13T12:51:56.741 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 12:51:56 smithi042 sudo[43514]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T12:51:56.741 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 12:51:56 smithi042 sudo[43514]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:56.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 12:51:56 smithi042 sudo[43514]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:57.211 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:56 smithi042 ceph-mon[23723]: pgmap v80: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:51:57.211 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:56 smithi042 ceph-mon[23723]: osdmap e23: 3 total, 3 up, 3 in 2024-09-13T12:51:57.211 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:57.212 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:56 smithi042 sudo[43537]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T12:51:57.212 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:56 smithi042 sudo[43537]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:57.212 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:56 smithi042 sudo[43537]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:56 smithi046 ceph-mon[32268]: pgmap v80: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:51:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:56 smithi046 ceph-mon[32268]: osdmap e23: 3 total, 3 up, 3 in 2024-09-13T12:51:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:57.478 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:57 smithi042 sudo[43559]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T12:51:57.479 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:57 smithi042 sudo[43559]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:57.479 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 12:51:57 smithi042 sudo[43559]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:57.992 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 12:51:57 smithi042 sudo[43567]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T12:51:57.992 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 12:51:57 smithi042 sudo[43567]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:57.993 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 12:51:57 smithi042 sudo[43567]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:58.491 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 12:51:58 smithi042 sudo[43629]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T12:51:58.491 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 12:51:58 smithi042 sudo[43629]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:58.491 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 12:51:58 smithi042 sudo[43629]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:58.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:58 smithi042 sudo[43675]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T12:51:58.794 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:58 smithi042 sudo[43675]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:58.794 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:58 smithi042 sudo[43675]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 sudo[43693]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T12:51:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 sudo[43693]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 sudo[43693]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:51:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: mgrmap e19: smithi042.jhsfyp(active, since 2m), standbys: smithi046.hcgddm 2024-09-13T12:51:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:59.416 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:51:59 smithi042 ceph-mon[23723]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: mgrmap e19: smithi042.jhsfyp(active, since 2m), standbys: smithi046.hcgddm 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.592 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:51:59.593 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 ceph-mon[32268]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T12:51:59.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 sudo[34614]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-13T12:51:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 sudo[34614]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:51:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:51:59 smithi046 sudo[34614]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 sudo[34617]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 sudo[34617]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 sudo[34617]: pam_unix(sudo:session): session closed for user root 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: pgmap v82: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi042 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi042 to 4289M 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='client.14394 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T12:52:00.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:52:00.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:52:00.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T12:52:00.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:00.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: pgmap v82: 1 pgs: 1 creating+activating; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi042 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi042 to 4289M 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='client.14394 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi042:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T12:52:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:01.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:01 smithi042 ceph-mon[23723]: pgmap v83: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:01.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:01 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1943332571' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a649835b-3f3c-4831-b752-da0a8a155794"}]: dispatch 2024-09-13T12:52:01.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:01 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1943332571' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a649835b-3f3c-4831-b752-da0a8a155794"}]': finished 2024-09-13T12:52:01.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:01 smithi042 ceph-mon[23723]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T12:52:01.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:01 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:01 smithi046 ceph-mon[32268]: pgmap v83: 1 pgs: 1 creating+activating; 0 B data, 17 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:01 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1943332571' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a649835b-3f3c-4831-b752-da0a8a155794"}]: dispatch 2024-09-13T12:52:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:01 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1943332571' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a649835b-3f3c-4831-b752-da0a8a155794"}]': finished 2024-09-13T12:52:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:01 smithi046 ceph-mon[32268]: osdmap e24: 4 total, 3 up, 4 in 2024-09-13T12:52:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:01 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:02.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:02 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/458097105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:52:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:02 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/458097105' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:52:03.482 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:03 smithi042 ceph-mon[23723]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:03 smithi046 ceph-mon[32268]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:05.419 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:05 smithi046 ceph-mon[32268]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:05.481 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:05 smithi042 ceph-mon[23723]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:07.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:07 smithi042 ceph-mon[23723]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:07.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:07 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T12:52:07.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:07 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:07.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:07 smithi046 ceph-mon[32268]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:07.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:07 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T12:52:07.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:07 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:08.486 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:08 smithi042 ceph-mon[23723]: Deploying daemon osd.3 on smithi042 2024-09-13T12:52:08.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:08 smithi046 ceph-mon[32268]: Deploying daemon osd.3 on smithi042 2024-09-13T12:52:09.288 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:09 smithi042 ceph-mon[23723]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:09.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:09 smithi046 ceph-mon[32268]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:11.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:11 smithi042 ceph-mon[23723]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:11 smithi046 ceph-mon[32268]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:13.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:13 smithi042 ceph-mon[23723]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:13.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:13 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:13.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:13 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:13.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:13 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:13.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:13 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:13 smithi046 ceph-mon[32268]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:13 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:13 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:13 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:13 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:15.292 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:15 smithi042 ceph-mon[23723]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:15 smithi046 ceph-mon[32268]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:16.045 INFO:teuthology.orchestra.run.smithi042.stdout:Created osd(s) 3 on host 'smithi042' 2024-09-13T12:52:16.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:16 smithi042 ceph-mon[23723]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T12:52:16.361 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:16 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:16 smithi046 ceph-mon[32268]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T12:52:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:16 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:16.669 DEBUG:teuthology.orchestra.run.smithi042:osd.3> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T12:52:16.672 INFO:tasks.cephadm:Deploying osd.4 on smithi046 with /dev/vg_nvme/lv_4... 2024-09-13T12:52:16.672 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-13T12:52:17.311 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:17 smithi046 ceph-mon[32268]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:17.311 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:17 smithi046 ceph-mon[32268]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T12:52:17.311 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:17 smithi046 ceph-mon[32268]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T12:52:17.311 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:17 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:17.311 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:17 smithi046 ceph-mon[32268]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:52:17.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-mon[23723]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:17.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-mon[23723]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T12:52:17.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-mon[23723]: osdmap e25: 4 total, 3 up, 4 in 2024-09-13T12:52:17.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:17.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-mon[23723]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]: dispatch 2024-09-13T12:52:17.492 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[45207]: 2024-09-13T12:52:17.205+0000 7f12056e3700 -1 osd.3 0 waiting for initial osdmap 2024-09-13T12:52:17.493 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 12:52:17 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[45207]: 2024-09-13T12:52:17.209+0000 7f120207e700 -1 osd.3 26 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T12:52:18.338 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:18 smithi046 ceph-mon[32268]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:52:18.338 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:18 smithi046 ceph-mon[32268]: osdmap e26: 4 total, 3 up, 4 in 2024-09-13T12:52:18.338 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:18 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:18.338 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:18 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:18.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:18 smithi042 ceph-mon[23723]: from='osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]}]': finished 2024-09-13T12:52:18.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:18 smithi042 ceph-mon[23723]: osdmap e26: 4 total, 3 up, 4 in 2024-09-13T12:52:18.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:18 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:18.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:18 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240] boot 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:19.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:19.512 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:52:19.512 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:52:19.512 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-09-13T12:52:19.513 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:19.513 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: osd.3 [v2:172.21.15.42:6826/536044240,v1:172.21.15.42:6827/536044240] boot 2024-09-13T12:52:19.513 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: osdmap e27: 4 total, 4 up, 4 in 2024-09-13T12:52:19.513 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:52:19.513 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:19.518 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:52:19.542 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_4 2024-09-13T12:52:20.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:20 smithi042 ceph-mon[23723]: osdmap e28: 4 total, 4 up, 4 in 2024-09-13T12:52:20.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:20 smithi046 ceph-mon[32268]: osdmap e28: 4 total, 4 up, 4 in 2024-09-13T12:52:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:21 smithi046 ceph-mon[32268]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:21.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:21 smithi042 ceph-mon[23723]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:22.335 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:52:22.335 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:52:22.335 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:52:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:52:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:23.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:23.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:23.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:23 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='client.24139 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:52:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:23.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:23 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi042 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi042 to 3217M 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3426686799' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb8dfb-dae1-486b-b971-5d54e5ac358c"}]: dispatch 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb8dfb-dae1-486b-b971-5d54e5ac358c"}]: dispatch 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eefb8dfb-dae1-486b-b971-5d54e5ac358c"}]': finished 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T12:52:24.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:24 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi042 2024-09-13T12:52:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi042 to 3217M 2024-09-13T12:52:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3426686799' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb8dfb-dae1-486b-b971-5d54e5ac358c"}]: dispatch 2024-09-13T12:52:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eefb8dfb-dae1-486b-b971-5d54e5ac358c"}]: dispatch 2024-09-13T12:52:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eefb8dfb-dae1-486b-b971-5d54e5ac358c"}]': finished 2024-09-13T12:52:24.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: osdmap e29: 5 total, 4 up, 5 in 2024-09-13T12:52:24.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:24 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:25 smithi042 ceph-mon[23723]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:25 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/430521675' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:52:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:25 smithi046 ceph-mon[32268]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:25 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/430521675' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:52:27.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:27 smithi042 ceph-mon[23723]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:27.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:27 smithi046 ceph-mon[32268]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:29.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:29 smithi042 ceph-mon[23723]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:29 smithi046 ceph-mon[32268]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:30.521 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:30 smithi046 ceph-mon[32268]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:30.521 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T12:52:30.521 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:30.521 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:30 smithi046 ceph-mon[32268]: Deploying daemon osd.4 on smithi046 2024-09-13T12:52:30.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:30 smithi042 ceph-mon[23723]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:30.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T12:52:30.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:30.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:30 smithi042 ceph-mon[23723]: Deploying daemon osd.4 on smithi046 2024-09-13T12:52:32.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:32 smithi046 ceph-mon[32268]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:32.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:32 smithi042 ceph-mon[23723]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:34.710 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:34 smithi046 ceph-mon[32268]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:34.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:34 smithi042 ceph-mon[23723]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:52:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:52:36.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:52:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:52:37.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:37 smithi046 ceph-mon[32268]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:37.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:37 smithi042 ceph-mon[23723]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:37.833 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 4 on host 'smithi046' 2024-09-13T12:52:38.712 DEBUG:teuthology.orchestra.run.smithi046:osd.4> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T12:52:38.715 INFO:tasks.cephadm:Deploying osd.5 on smithi046 with /dev/vg_nvme/lv_3... 2024-09-13T12:52:38.715 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-13T12:52:39.000 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:38 smithi046 ceph-mon[32268]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:39.000 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:38 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:39.000 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:38 smithi046 ceph-mon[32268]: from='osd.4 [v2:172.21.15.46:6800/3781993651,v1:172.21.15.46:6801/3781993651]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T12:52:39.000 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:38 smithi046 ceph-mon[32268]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T12:52:39.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:38 smithi042 ceph-mon[23723]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:39.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:38 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:39.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:38 smithi042 ceph-mon[23723]: from='osd.4 [v2:172.21.15.46:6800/3781993651,v1:172.21.15.46:6801/3781993651]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T12:52:39.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:38 smithi042 ceph-mon[23723]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T12:52:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-mon[32268]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T12:52:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-mon[32268]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T12:52:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-mon[32268]: from='osd.4 [v2:172.21.15.46:6800/3781993651,v1:172.21.15.46:6801/3781993651]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:52:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-mon[32268]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:52:40.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:40.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[36533]: 2024-09-13T12:52:40.040+0000 7fe1a64ab700 -1 osd.4 0 waiting for initial osdmap 2024-09-13T12:52:40.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 12:52:40 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[36533]: 2024-09-13T12:52:40.045+0000 7fe1a0e42700 -1 osd.4 31 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T12:52:40.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:40 smithi042 ceph-mon[23723]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T12:52:40.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:40 smithi042 ceph-mon[23723]: osdmap e30: 5 total, 4 up, 5 in 2024-09-13T12:52:40.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:40 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:40.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:40 smithi042 ceph-mon[23723]: from='osd.4 [v2:172.21.15.46:6800/3781993651,v1:172.21.15.46:6801/3781993651]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:52:40.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:40 smithi042 ceph-mon[23723]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:52:40.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:40 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:41.296 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:41 smithi046 ceph-mon[32268]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:41.296 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:41 smithi046 ceph-mon[32268]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:52:41.296 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:41 smithi046 ceph-mon[32268]: osdmap e31: 5 total, 4 up, 5 in 2024-09-13T12:52:41.296 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:41 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:41.296 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:41 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:41.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:41 smithi042 ceph-mon[23723]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:41.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:41 smithi042 ceph-mon[23723]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:52:41.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:41 smithi042 ceph-mon[23723]: osdmap e31: 5 total, 4 up, 5 in 2024-09-13T12:52:41.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:41 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:41.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:41 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:42.234 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:42 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:52:42.234 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:42 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:52:42.234 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:42 smithi046 ceph-mon[32268]: osd.4 [v2:172.21.15.46:6800/3781993651,v1:172.21.15.46:6801/3781993651] boot 2024-09-13T12:52:42.234 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:42 smithi046 ceph-mon[32268]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T12:52:42.234 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:42.240 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:52:42.265 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_3 2024-09-13T12:52:42.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:42 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:52:42.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:42 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:52:42.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:42 smithi042 ceph-mon[23723]: osd.4 [v2:172.21.15.46:6800/3781993651,v1:172.21.15.46:6801/3781993651] boot 2024-09-13T12:52:42.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:42 smithi042 ceph-mon[23723]: osdmap e32: 5 total, 5 up, 5 in 2024-09-13T12:52:42.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:52:43.299 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:43 smithi046 ceph-mon[32268]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:43.299 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:43 smithi046 ceph-mon[32268]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T12:52:43.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:43 smithi042 ceph-mon[23723]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-09-13T12:52:43.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:43 smithi042 ceph-mon[23723]: osdmap e33: 5 total, 5 up, 5 in 2024-09-13T12:52:44.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:44 smithi046 ceph-mon[32268]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T12:52:44.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:44 smithi042 ceph-mon[23723]: osdmap e34: 5 total, 5 up, 5 in 2024-09-13T12:52:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:52:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T12:52:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:45.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:45.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:45.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:45.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: osdmap e35: 5 total, 5 up, 5 in 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:52:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:52:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:46 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi046 2024-09-13T12:52:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:46 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi046 to 15940M 2024-09-13T12:52:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:46 smithi046 ceph-mon[32268]: osdmap e36: 5 total, 5 up, 5 in 2024-09-13T12:52:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:52:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:52:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:46 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:46.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:46 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi046 2024-09-13T12:52:46.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:46 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi046 to 15940M 2024-09-13T12:52:46.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:46 smithi042 ceph-mon[23723]: osdmap e36: 5 total, 5 up, 5 in 2024-09-13T12:52:46.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:52:46.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:52:46.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:46 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:47 smithi046 ceph-mon[32268]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T12:52:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:47 smithi046 ceph-mon[32268]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:52:47.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:47 smithi042 ceph-mon[23723]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T12:52:47.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:47 smithi042 ceph-mon[23723]: from='client.24157 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:52:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:48 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2420638284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7785b24e-0c12-4283-9572-645442edc6ce"}]: dispatch 2024-09-13T12:52:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:48 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7785b24e-0c12-4283-9572-645442edc6ce"}]: dispatch 2024-09-13T12:52:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:48 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7785b24e-0c12-4283-9572-645442edc6ce"}]': finished 2024-09-13T12:52:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:48 smithi046 ceph-mon[32268]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T12:52:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:48 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:52:48.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:48 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2420638284' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7785b24e-0c12-4283-9572-645442edc6ce"}]: dispatch 2024-09-13T12:52:48.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:48 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7785b24e-0c12-4283-9572-645442edc6ce"}]: dispatch 2024-09-13T12:52:48.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:48 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7785b24e-0c12-4283-9572-645442edc6ce"}]': finished 2024-09-13T12:52:48.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:48 smithi042 ceph-mon[23723]: osdmap e37: 6 total, 5 up, 6 in 2024-09-13T12:52:48.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:48 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:52:49.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:49 smithi046 ceph-mon[32268]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-13T12:52:49.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:49 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2592752781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:52:49.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:49 smithi042 ceph-mon[23723]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-09-13T12:52:49.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:49 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2592752781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:52:51.278 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:51 smithi046 ceph-mon[32268]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T12:52:51.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:51 smithi042 ceph-mon[23723]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T12:52:53.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:53 smithi046 ceph-mon[32268]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-13T12:52:53.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:53 smithi042 ceph-mon[23723]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-09-13T12:52:55.333 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:55 smithi046 ceph-mon[32268]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T12:52:55.448 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T12:52:55.448 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:55 smithi042 ceph-mon[23723]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T12:52:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T12:52:55.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:52:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:56 smithi046 ceph-mon[32268]: Deploying daemon osd.5 on smithi046 2024-09-13T12:52:56.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:56 smithi042 ceph-mon[23723]: Deploying daemon osd.5 on smithi046 2024-09-13T12:52:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:57 smithi046 ceph-mon[32268]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:52:57.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:57 smithi042 ceph-mon[23723]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:52:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:52:59 smithi046 ceph-mon[32268]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:52:59.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:52:59 smithi042 ceph-mon[23723]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:00 smithi042 ceph-mon[23723]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:00.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:00 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:00.774 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:00 smithi046 ceph-mon[32268]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:00.775 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:00.775 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:00.775 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:00.775 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:00 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:02.506 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:02 smithi046 ceph-mon[32268]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:02.637 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 5 on host 'smithi046' 2024-09-13T12:53:02.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:02 smithi042 ceph-mon[23723]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:03.244 DEBUG:teuthology.orchestra.run.smithi046:osd.5> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T12:53:03.247 INFO:tasks.cephadm:Deploying osd.6 on smithi046 with /dev/vg_nvme/lv_2... 2024-09-13T12:53:03.247 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-13T12:53:03.497 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:03 smithi046 ceph-mon[32268]: from='osd.5 [v2:172.21.15.46:6808/2495505474,v1:172.21.15.46:6809/2495505474]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T12:53:03.497 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:03 smithi046 ceph-mon[32268]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T12:53:03.497 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:03 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:03.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:03 smithi042 ceph-mon[23723]: from='osd.5 [v2:172.21.15.46:6808/2495505474,v1:172.21.15.46:6809/2495505474]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T12:53:03.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:03 smithi042 ceph-mon[23723]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T12:53:03.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:03 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-mon[32268]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-mon[32268]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T12:53:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-mon[32268]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T12:53:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-mon[32268]: from='osd.5 [v2:172.21.15.46:6808/2495505474,v1:172.21.15.46:6809/2495505474]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-mon[32268]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:04.618 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[39498]: 2024-09-13T12:53:04.461+0000 7f89e713e700 -1 osd.5 0 waiting for initial osdmap 2024-09-13T12:53:04.618 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 12:53:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[39498]: 2024-09-13T12:53:04.467+0000 7f89e1ad5700 -1 osd.5 39 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T12:53:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:04 smithi042 ceph-mon[23723]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:04 smithi042 ceph-mon[23723]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T12:53:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:04 smithi042 ceph-mon[23723]: osdmap e38: 6 total, 5 up, 6 in 2024-09-13T12:53:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:04 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:04 smithi042 ceph-mon[23723]: from='osd.5 [v2:172.21.15.46:6808/2495505474,v1:172.21.15.46:6809/2495505474]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:04 smithi042 ceph-mon[23723]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:05.499 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:05 smithi046 ceph-mon[32268]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:53:05.500 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:05 smithi046 ceph-mon[32268]: osdmap e39: 6 total, 5 up, 6 in 2024-09-13T12:53:05.500 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:05 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:05.500 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:05 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:05.500 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:05 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:05.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:05 smithi042 ceph-mon[23723]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:53:05.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:05 smithi042 ceph-mon[23723]: osdmap e39: 6 total, 5 up, 6 in 2024-09-13T12:53:05.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:05 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:05.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:05 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:05.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:05 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: osd.5 [v2:172.21.15.46:6808/2495505474,v1:172.21.15.46:6809/2495505474] boot 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T12:53:06.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:06 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: osd.5 [v2:172.21.15.46:6808/2495505474,v1:172.21.15.46:6809/2495505474] boot 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: osdmap e40: 6 total, 6 up, 6 in 2024-09-13T12:53:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:06 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:53:07.315 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:53:07.348 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_2 2024-09-13T12:53:07.581 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:07 smithi046 ceph-mon[32268]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T12:53:07.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:07 smithi042 ceph-mon[23723]: osdmap e41: 6 total, 6 up, 6 in 2024-09-13T12:53:08.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:08 smithi042 ceph-mon[23723]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:08.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:08 smithi042 ceph-mon[23723]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T12:53:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:08 smithi046 ceph-mon[32268]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:08 smithi046 ceph-mon[32268]: osdmap e42: 6 total, 6 up, 6 in 2024-09-13T12:53:09.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:09 smithi042 ceph-mon[23723]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T12:53:09.830 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:09 smithi046 ceph-mon[32268]: osdmap e43: 6 total, 6 up, 6 in 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: pgmap v138: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: osdmap e44: 6 total, 6 up, 6 in 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi046 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi046 to 7970M 2024-09-13T12:53:10.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:10.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:10.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:10.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:10 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: pgmap v138: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: osdmap e44: 6 total, 6 up, 6 in 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi046 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi046 to 7970M 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:10 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:12.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:12 smithi042 ceph-mon[23723]: pgmap v140: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:12 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/2186072327' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a977a6a5-0770-47b2-beea-ad49b4c0227f"}]: dispatch 2024-09-13T12:53:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:12 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a977a6a5-0770-47b2-beea-ad49b4c0227f"}]: dispatch 2024-09-13T12:53:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:12 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a977a6a5-0770-47b2-beea-ad49b4c0227f"}]': finished 2024-09-13T12:53:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:12 smithi042 ceph-mon[23723]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T12:53:12.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:12 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:12 smithi046 ceph-mon[32268]: pgmap v140: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:12 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/2186072327' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a977a6a5-0770-47b2-beea-ad49b4c0227f"}]: dispatch 2024-09-13T12:53:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:12 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a977a6a5-0770-47b2-beea-ad49b4c0227f"}]: dispatch 2024-09-13T12:53:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:12 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a977a6a5-0770-47b2-beea-ad49b4c0227f"}]': finished 2024-09-13T12:53:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:12 smithi046 ceph-mon[32268]: osdmap e45: 7 total, 6 up, 7 in 2024-09-13T12:53:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:12 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:13.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:13 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3040836313' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:53:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:13 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3040836313' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:53:14.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:14 smithi042 ceph-mon[23723]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:14 smithi046 ceph-mon[32268]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:16.709 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:16 smithi046 ceph-mon[32268]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:16.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:16 smithi042 ceph-mon[23723]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:18.819 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:18 smithi046 ceph-mon[32268]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:18.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:18 smithi042 ceph-mon[23723]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T12:53:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:19 smithi046 ceph-mon[32268]: Deploying daemon osd.6 on smithi046 2024-09-13T12:53:19.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T12:53:19.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:19.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:19 smithi042 ceph-mon[23723]: Deploying daemon osd.6 on smithi046 2024-09-13T12:53:20.566 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:20 smithi046 ceph-mon[32268]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:20.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:20 smithi042 ceph-mon[23723]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:22 smithi046 ceph-mon[32268]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:22.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:22 smithi042 ceph-mon[23723]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:25 smithi046 ceph-mon[32268]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:25.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:25 smithi042 ceph-mon[23723]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:25.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:27.168 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:27 smithi046 ceph-mon[32268]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:27.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:27 smithi042 ceph-mon[23723]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:28.116 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 6 on host 'smithi046' 2024-09-13T12:53:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:28 smithi046 ceph-mon[32268]: from='osd.6 [v2:172.21.15.46:6816/3075368475,v1:172.21.15.46:6817/3075368475]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T12:53:28.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:28 smithi046 ceph-mon[32268]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T12:53:28.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:28 smithi042 ceph-mon[23723]: from='osd.6 [v2:172.21.15.46:6816/3075368475,v1:172.21.15.46:6817/3075368475]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T12:53:28.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:28 smithi042 ceph-mon[23723]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T12:53:28.691 DEBUG:teuthology.orchestra.run.smithi046:osd.6> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T12:53:28.694 INFO:tasks.cephadm:Deploying osd.7 on smithi046 with /dev/vg_nvme/lv_1... 2024-09-13T12:53:28.694 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-13T12:53:29.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:29.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T12:53:29.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T12:53:29.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:29.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: from='osd.6 [v2:172.21.15.46:6816/3075368475,v1:172.21.15.46:6817/3075368475]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:29.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:29.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:29.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:29.368 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[42457]: 2024-09-13T12:53:29.088+0000 7f50f5e3c700 -1 osd.6 0 waiting for initial osdmap 2024-09-13T12:53:29.368 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 12:53:29 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[42457]: 2024-09-13T12:53:29.096+0000 7f50f07d3700 -1 osd.6 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T12:53:29.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: osdmap e46: 7 total, 6 up, 7 in 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: from='osd.6 [v2:172.21.15.46:6816/3075368475,v1:172.21.15.46:6817/3075368475]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:29.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:29 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: osdmap e47: 7 total, 6 up, 7 in 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: osd.6 [v2:172.21.15.46:6816/3075368475,v1:172.21.15.46:6817/3075368475] boot 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T12:53:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:30 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: osdmap e47: 7 total, 6 up, 7 in 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: osd.6 [v2:172.21.15.46:6816/3075368475,v1:172.21.15.46:6817/3075368475] boot 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: osdmap e48: 7 total, 7 up, 7 in 2024-09-13T12:53:30.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:30 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:53:31.315 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:31 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:53:31.315 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:31 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:53:31.316 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:31 smithi046 ceph-mon[32268]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:31.316 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:31 smithi046 ceph-mon[32268]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T12:53:31.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:31 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:53:31.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:31 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:53:31.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:31 smithi042 ceph-mon[23723]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-09-13T12:53:31.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:31 smithi042 ceph-mon[23723]: osdmap e49: 7 total, 7 up, 7 in 2024-09-13T12:53:32.697 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:32 smithi046 ceph-mon[32268]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:32.697 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:32 smithi046 ceph-mon[32268]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T12:53:32.703 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-09-13T12:53:32.730 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_1 2024-09-13T12:53:32.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:32 smithi042 ceph-mon[23723]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:32.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:32 smithi042 ceph-mon[23723]: osdmap e50: 7 total, 7 up, 7 in 2024-09-13T12:53:34.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:34 smithi046 ceph-mon[32268]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T12:53:34.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:34 smithi042 ceph-mon[23723]: osdmap e51: 7 total, 7 up, 7 in 2024-09-13T12:53:35.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:35.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: osdmap e52: 7 total, 7 up, 7 in 2024-09-13T12:53:35.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:35.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: osdmap e52: 7 total, 7 up, 7 in 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:35.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:36.360 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi046 2024-09-13T12:53:36.361 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi046 to 5313M 2024-09-13T12:53:36.361 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:53:36.361 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:53:36.361 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:53:36.361 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:53:36.361 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:36.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi046 2024-09-13T12:53:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi046 to 5313M 2024-09-13T12:53:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:53:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:53:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T12:53:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T12:53:36.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:37.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:37 smithi046 ceph-mon[32268]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T12:53:37.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:37 smithi046 ceph-mon[32268]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:53:37.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:37 smithi042 ceph-mon[23723]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2024-09-13T12:53:37.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:37 smithi042 ceph-mon[23723]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:53:39.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T12:53:39.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/3841317921' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6ba631db-d6a4-4408-92de-9bf900a99dc9"}]: dispatch 2024-09-13T12:53:39.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6ba631db-d6a4-4408-92de-9bf900a99dc9"}]: dispatch 2024-09-13T12:53:39.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6ba631db-d6a4-4408-92de-9bf900a99dc9"}]': finished 2024-09-13T12:53:39.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T12:53:39.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:39.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:39 smithi046 ceph-mon[32268]: from='client.? 172.21.15.46:0/350432446' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/3841317921' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6ba631db-d6a4-4408-92de-9bf900a99dc9"}]: dispatch 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6ba631db-d6a4-4408-92de-9bf900a99dc9"}]: dispatch 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6ba631db-d6a4-4408-92de-9bf900a99dc9"}]': finished 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: osdmap e53: 8 total, 7 up, 8 in 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:39.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:39 smithi042 ceph-mon[23723]: from='client.? 172.21.15.46:0/350432446' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T12:53:41.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:41 smithi046 ceph-mon[32268]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-13T12:53:41.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:41 smithi042 ceph-mon[23723]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-13T12:53:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:43 smithi046 ceph-mon[32268]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T12:53:43.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:43 smithi042 ceph-mon[23723]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-09-13T12:53:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:45 smithi046 ceph-mon[32268]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-13T12:53:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T12:53:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:45 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:45.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:45 smithi042 ceph-mon[23723]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-13T12:53:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T12:53:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:45 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:46.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:46 smithi042 ceph-mon[23723]: Deploying daemon osd.7 on smithi046 2024-09-13T12:53:46.497 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:46 smithi046 ceph-mon[32268]: Deploying daemon osd.7 on smithi046 2024-09-13T12:53:47.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:47 smithi042 ceph-mon[23723]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:47.538 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:47 smithi046 ceph-mon[32268]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:49.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:49 smithi042 ceph-mon[23723]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:49.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:49 smithi046 ceph-mon[32268]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:51.270 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:51 smithi046 ceph-mon[32268]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:51.270 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:51.270 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:51.270 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:51.270 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:51.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:51 smithi042 ceph-mon[23723]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:51.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:51.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:51.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:51.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:53.353 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:53 smithi046 ceph-mon[32268]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:53 smithi042 ceph-mon[23723]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:54.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:54 smithi046 ceph-mon[32268]: from='osd.7 [v2:172.21.15.46:6824/2174137873,v1:172.21.15.46:6825/2174137873]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T12:53:54.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:54 smithi046 ceph-mon[32268]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T12:53:54.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:54.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:54 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:54.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:54 smithi042 ceph-mon[23723]: from='osd.7 [v2:172.21.15.46:6824/2174137873,v1:172.21.15.46:6825/2174137873]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T12:53:54.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:54 smithi042 ceph-mon[23723]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T12:53:54.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:54.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:54 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:54.989 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 7 on host 'smithi046' 2024-09-13T12:53:55.293 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='osd.7 [v2:172.21.15.46:6824/2174137873,v1:172.21.15.46:6825/2174137873]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:55.294 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:55 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: osdmap e54: 8 total, 7 up, 8 in 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='osd.7 [v2:172.21.15.46:6824/2174137873,v1:172.21.15.46:6825/2174137873]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:53:55.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:55 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:53:55.618 DEBUG:teuthology.orchestra.run.smithi046:osd.7> sudo journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T12:53:55.620 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T12:53:55.620 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T12:53:56.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:56 smithi042 ceph-mon[23723]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:53:56.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:56 smithi042 ceph-mon[23723]: osdmap e55: 8 total, 7 up, 8 in 2024-09-13T12:53:56.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:56.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:56.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:56 smithi046 ceph-mon[32268]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-09-13T12:53:56.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:56 smithi046 ceph-mon[32268]: osdmap e55: 8 total, 7 up, 8 in 2024-09-13T12:53:56.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:56.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: purged_snaps scrub starts 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: purged_snaps scrub ok 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: osd.7 [v2:172.21.15.46:6824/2174137873,v1:172.21.15.46:6825/2174137873] boot 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T12:53:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:57.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: purged_snaps scrub starts 2024-09-13T12:53:57.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: purged_snaps scrub ok 2024-09-13T12:53:57.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-09-13T12:53:57.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:57.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: osd.7 [v2:172.21.15.46:6824/2174137873,v1:172.21.15.46:6825/2174137873] boot 2024-09-13T12:53:57.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: osdmap e56: 8 total, 8 up, 8 in 2024-09-13T12:53:57.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:53:57.588 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:53:58.181 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1726232036,"num_in_osds":8,"osd_in_since":1726232018,"num_remapped_pgs":1} 2024-09-13T12:53:58.182 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T12:53:58.403 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:53:58.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:58 smithi042 ceph-mon[23723]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T12:53:58.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:58 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/607628516' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T12:53:58.542 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:58 smithi046 ceph-mon[32268]: osdmap e57: 8 total, 8 up, 8 in 2024-09-13T12:53:58.542 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:58 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/607628516' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:59.415 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:53:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:53:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: osdmap e58: 8 total, 8 up, 8 in 2024-09-13T12:53:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:53:59.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:53:59.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:53:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:54:00.098 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:00.098 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":59,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","created":"2024-09-13T12:44:58.813707+0000","modified":"2024-09-13T12:53:59.154968+0000","last_up_change":"2024-09-13T12:53:56.141100+0000","last_in_change":"2024-09-13T12:53:38.327038+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T12:51:53.873752+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"edec8271-f2f4-4f2b-b7b8-710f44c128fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6803","nonce":3230857529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6805","nonce":3230857529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6809","nonce":3230857529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6807","nonce":3230857529}]},"public_addr":"172.21.15.42:6803/3230857529","cluster_addr":"172.21.15.42:6805/3230857529","heartbeat_back_addr":"172.21.15.42:6809/3230857529","heartbeat_front_addr":"172.21.15.42:6807/3230857529","state":["exists","up"]},{"osd":1,"uuid":"a68de12b-5ed6-474f-9a88-08d119714e95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6811","nonce":1160947428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6813","nonce":1160947428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6817","nonce":1160947428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6815","nonce":1160947428}]},"public_addr":"172.21.15.42:6811/1160947428","cluster_addr":"172.21.15.42:6813/1160947428","heartbeat_back_addr":"172.21.15.42:6817/1160947428","heartbeat_front_addr":"172.21.15.42:6815/1160947428","state":["exists","up"]},{"osd":2,"uuid":"0fa5feee-949c-46a9-8edc-5bc91cbcbe0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6819","nonce":912984957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6821","nonce":912984957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6825","nonce":912984957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6823","nonce":912984957}]},"public_addr":"172.21.15.42:6819/912984957","cluster_addr":"172.21.15.42:6821/912984957","heartbeat_back_addr":"172.21.15.42:6825/912984957","heartbeat_front_addr":"172.21.15.42:6823/912984957","state":["exists","up"]},{"osd":3,"uuid":"a649835b-3f3c-4831-b752-da0a8a155794","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6827","nonce":536044240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6829","nonce":536044240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6833","nonce":536044240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6831","nonce":536044240}]},"public_addr":"172.21.15.42:6827/536044240","cluster_addr":"172.21.15.42:6829/536044240","heartbeat_back_addr":"172.21.15.42:6833/536044240","heartbeat_front_addr":"172.21.15.42:6831/536044240","state":["exists","up"]},{"osd":4,"uuid":"eefb8dfb-dae1-486b-b971-5d54e5ac358c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6800","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6801","nonce":3781993651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6802","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6803","nonce":3781993651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6806","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6807","nonce":3781993651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6804","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6805","nonce":3781993651}]},"public_addr":"172.21.15.46:6801/3781993651","cluster_addr":"172.21.15.46:6803/3781993651","heartbeat_back_addr":"172.21.15.46:6807/3781993651","heartbeat_front_addr":"172.21.15.46:6805/3781993651","state":["exists","up"]},{"osd":5,"uuid":"7785b24e-0c12-4283-9572-645442edc6ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6808","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6809","nonce":2495505474}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6810","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6811","nonce":2495505474}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6814","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6815","nonce":2495505474}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6812","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6813","nonce":2495505474}]},"public_addr":"172.21.15.46:6809/2495505474","cluster_addr":"172.21.15.46:6811/2495505474","heartbeat_back_addr":"172.21.15.46:6815/2495505474","heartbeat_front_addr":"172.21.15.46:6813/2495505474","state":["exists","up"]},{"osd":6,"uuid":"a977a6a5-0770-47b2-beea-ad49b4c0227f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6816","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6817","nonce":3075368475}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6818","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6819","nonce":3075368475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6822","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6823","nonce":3075368475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6820","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6821","nonce":3075368475}]},"public_addr":"172.21.15.46:6817/3075368475","cluster_addr":"172.21.15.46:6819/3075368475","heartbeat_back_addr":"172.21.15.46:6823/3075368475","heartbeat_front_addr":"172.21.15.46:6821/3075368475","state":["exists","up"]},{"osd":7,"uuid":"6ba631db-d6a4-4408-92de-9bf900a99dc9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6824","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6825","nonce":2174137873}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6826","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6827","nonce":2174137873}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6830","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6831","nonce":2174137873}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6828","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6829","nonce":2174137873}]},"public_addr":"172.21.15.46:6825/2174137873","cluster_addr":"172.21.15.46:6827/2174137873","heartbeat_back_addr":"172.21.15.46:6831/2174137873","heartbeat_front_addr":"172.21.15.46:6829/2174137873","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:50:51.202792+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:51:22.490251+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:51:49.558149+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:52:16.288067+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:52:39.176258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:03.484085+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:28.136067+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:54.801101+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/154408839":"2024-09-14T12:49:34.486008+0000","172.21.15.42:0/1972800177":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6800/3161296684":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6801/273214107":"2024-09-14T12:45:57.248998+0000","172.21.15.42:6800/273214107":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/1533562927":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/2634043947":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/2757003853":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6801/2424236657":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/3321752004":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/207953028":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/3801247064":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/2636830803":"2024-09-14T12:45:26.746626+0000","172.21.15.42:6801/3161296684":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6800/2424236657":"2024-09-14T12:45:26.746626+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T12:54:00.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:00 smithi042 ceph-mon[23723]: Detected new or changed devices on smithi046 2024-09-13T12:54:00.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:00 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi046 to 3985M 2024-09-13T12:54:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:00 smithi042 ceph-mon[23723]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T12:54:00.374 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:00 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2613467447' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T12:54:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:00 smithi046 ceph-mon[32268]: Detected new or changed devices on smithi046 2024-09-13T12:54:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:00 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi046 to 3985M 2024-09-13T12:54:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:00 smithi046 ceph-mon[32268]: osdmap e59: 8 total, 8 up, 8 in 2024-09-13T12:54:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:00 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2613467447' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T12:54:00.747 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T12:51:53.873752+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '23', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T12:54:00.747 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T12:54:00.971 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:01.347 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:01 smithi042 ceph-mon[23723]: pgmap v179: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T12:54:01.347 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:01 smithi042 ceph-mon[23723]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T12:54:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:01 smithi046 ceph-mon[32268]: pgmap v179: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s, 0 objects/s recovering 2024-09-13T12:54:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:01 smithi046 ceph-mon[32268]: osdmap e60: 8 total, 8 up, 8 in 2024-09-13T12:54:02.688 INFO:teuthology.orchestra.run.smithi042.stdout:pg_num: 1 2024-09-13T12:54:03.301 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T12:54:03.301 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T12:54:03.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:03 smithi042 ceph-mon[23723]: pgmap v181: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 121 KiB/s, 0 objects/s recovering 2024-09-13T12:54:03.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:03 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/4101244319' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T12:54:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:03 smithi046 ceph-mon[32268]: pgmap v181: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 121 KiB/s, 0 objects/s recovering 2024-09-13T12:54:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:03 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/4101244319' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T12:54:05.293 INFO:teuthology.orchestra.run.smithi042.stdout:[client.0] 2024-09-13T12:54:05.294 INFO:teuthology.orchestra.run.smithi042.stdout: key = AQDtNeRmYANFERAA29lGOMQQ3AX3W47uhM45+Q== 2024-09-13T12:54:05.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:05 smithi042 ceph-mon[23723]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T12:54:05.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:05 smithi046 ceph-mon[32268]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-09-13T12:54:05.910 DEBUG:teuthology.orchestra.run.smithi042:> set -ex 2024-09-13T12:54:05.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T12:54:05.910 DEBUG:teuthology.orchestra.run.smithi042:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T12:54:05.949 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-13T12:54:06.201 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:06.482 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:06 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1916303064' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T12:54:06.482 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:06 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1916303064' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T12:54:06.600 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:06 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1916303064' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T12:54:06.600 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:06 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1916303064' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T12:54:07.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:07 smithi042 ceph-mon[23723]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T12:54:07.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:07 smithi046 ceph-mon[32268]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-09-13T12:54:07.944 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:08.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:08 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/389778504' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T12:54:08.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:08 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/389778504' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-13T12:54:08.620 INFO:teuthology.orchestra.run.smithi042.stdout:[{"version":21,"timestamp":"2024-09-13T12:53:58.580341+0000","name":"","changes":[{"name":"osd/host:smithi046/osd_memory_target","previous_value":"5571552597","new_value":"4178664448"}]}] 2024-09-13T12:54:08.621 INFO:tasks.ceph_manager:config epoch is 21 2024-09-13T12:54:08.621 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T12:54:08.621 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T12:54:08.621 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T12:54:08.841 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:09.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:09 smithi042 ceph-mon[23723]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:09.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:09 smithi046 ceph-mon[32268]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:10.618 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:11.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:11 smithi042 ceph-mon[23723]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:11.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:11 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2758707551' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T12:54:11.318 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":19,"active_gid":14300,"active_name":"smithi042.jhsfyp","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6800","nonce":2494771138},{"type":"v1","addr":"172.21.15.42:6801","nonce":2494771138}]},"active_addr":"172.21.15.42:6801/2494771138","active_change":"2024-09-13T12:49:34.486403+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":14322,"name":"smithi046.hcgddm","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.42:8443/","prometheus":"http://172.21.15.42:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":5,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":1302892198}]},{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":1769040698}]},{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":873609154}]},{"addrvec":[{"type":"v2","addr":"172.21.15.42:0","nonce":3612372344}]}]}} 2024-09-13T12:54:11.322 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T12:54:11.322 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T12:54:11.322 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T12:54:11.543 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:11 smithi046 ceph-mon[32268]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:11 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2758707551' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T12:54:13.239 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:13.239 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":60,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","created":"2024-09-13T12:44:58.813707+0000","modified":"2024-09-13T12:54:00.159698+0000","last_up_change":"2024-09-13T12:53:56.141100+0000","last_in_change":"2024-09-13T12:53:38.327038+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T12:51:53.873752+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"edec8271-f2f4-4f2b-b7b8-710f44c128fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6803","nonce":3230857529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6805","nonce":3230857529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6809","nonce":3230857529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6807","nonce":3230857529}]},"public_addr":"172.21.15.42:6803/3230857529","cluster_addr":"172.21.15.42:6805/3230857529","heartbeat_back_addr":"172.21.15.42:6809/3230857529","heartbeat_front_addr":"172.21.15.42:6807/3230857529","state":["exists","up"]},{"osd":1,"uuid":"a68de12b-5ed6-474f-9a88-08d119714e95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6811","nonce":1160947428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6813","nonce":1160947428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6817","nonce":1160947428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6815","nonce":1160947428}]},"public_addr":"172.21.15.42:6811/1160947428","cluster_addr":"172.21.15.42:6813/1160947428","heartbeat_back_addr":"172.21.15.42:6817/1160947428","heartbeat_front_addr":"172.21.15.42:6815/1160947428","state":["exists","up"]},{"osd":2,"uuid":"0fa5feee-949c-46a9-8edc-5bc91cbcbe0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6819","nonce":912984957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6821","nonce":912984957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6825","nonce":912984957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6823","nonce":912984957}]},"public_addr":"172.21.15.42:6819/912984957","cluster_addr":"172.21.15.42:6821/912984957","heartbeat_back_addr":"172.21.15.42:6825/912984957","heartbeat_front_addr":"172.21.15.42:6823/912984957","state":["exists","up"]},{"osd":3,"uuid":"a649835b-3f3c-4831-b752-da0a8a155794","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6827","nonce":536044240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6829","nonce":536044240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6833","nonce":536044240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6831","nonce":536044240}]},"public_addr":"172.21.15.42:6827/536044240","cluster_addr":"172.21.15.42:6829/536044240","heartbeat_back_addr":"172.21.15.42:6833/536044240","heartbeat_front_addr":"172.21.15.42:6831/536044240","state":["exists","up"]},{"osd":4,"uuid":"eefb8dfb-dae1-486b-b971-5d54e5ac358c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6800","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6801","nonce":3781993651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6802","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6803","nonce":3781993651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6806","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6807","nonce":3781993651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6804","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6805","nonce":3781993651}]},"public_addr":"172.21.15.46:6801/3781993651","cluster_addr":"172.21.15.46:6803/3781993651","heartbeat_back_addr":"172.21.15.46:6807/3781993651","heartbeat_front_addr":"172.21.15.46:6805/3781993651","state":["exists","up"]},{"osd":5,"uuid":"7785b24e-0c12-4283-9572-645442edc6ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6808","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6809","nonce":2495505474}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6810","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6811","nonce":2495505474}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6814","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6815","nonce":2495505474}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6812","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6813","nonce":2495505474}]},"public_addr":"172.21.15.46:6809/2495505474","cluster_addr":"172.21.15.46:6811/2495505474","heartbeat_back_addr":"172.21.15.46:6815/2495505474","heartbeat_front_addr":"172.21.15.46:6813/2495505474","state":["exists","up"]},{"osd":6,"uuid":"a977a6a5-0770-47b2-beea-ad49b4c0227f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6816","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6817","nonce":3075368475}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6818","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6819","nonce":3075368475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6822","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6823","nonce":3075368475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6820","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6821","nonce":3075368475}]},"public_addr":"172.21.15.46:6817/3075368475","cluster_addr":"172.21.15.46:6819/3075368475","heartbeat_back_addr":"172.21.15.46:6823/3075368475","heartbeat_front_addr":"172.21.15.46:6821/3075368475","state":["exists","up"]},{"osd":7,"uuid":"6ba631db-d6a4-4408-92de-9bf900a99dc9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6824","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6825","nonce":2174137873}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6826","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6827","nonce":2174137873}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6830","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6831","nonce":2174137873}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6828","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6829","nonce":2174137873}]},"public_addr":"172.21.15.46:6825/2174137873","cluster_addr":"172.21.15.46:6827/2174137873","heartbeat_back_addr":"172.21.15.46:6831/2174137873","heartbeat_front_addr":"172.21.15.46:6829/2174137873","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:50:51.202792+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:51:22.490251+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:51:49.558149+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:52:16.288067+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:52:39.176258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:03.484085+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:28.136067+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:54.801101+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/154408839":"2024-09-14T12:49:34.486008+0000","172.21.15.42:0/1972800177":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6800/3161296684":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6801/273214107":"2024-09-14T12:45:57.248998+0000","172.21.15.42:6800/273214107":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/1533562927":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/2634043947":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/2757003853":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6801/2424236657":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/3321752004":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/207953028":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/3801247064":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/2636830803":"2024-09-14T12:45:26.746626+0000","172.21.15.42:6801/3161296684":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6800/2424236657":"2024-09-14T12:45:26.746626+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T12:54:13.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:13 smithi042 ceph-mon[23723]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:13.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:13 smithi046 ceph-mon[32268]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:13.836 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T12:54:13.836 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T12:54:14.053 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:14.439 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:14 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/119829081' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T12:54:14.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:14 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/119829081' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T12:54:15.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:15 smithi042 ceph-mon[23723]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:15 smithi046 ceph-mon[32268]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:15.781 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:15.781 INFO:teuthology.orchestra.run.smithi042.stdout:{"epoch":60,"fsid":"e7c8d5a8-71cd-11ef-bceb-c7b262605968","created":"2024-09-13T12:44:58.813707+0000","modified":"2024-09-13T12:54:00.159698+0000","last_up_change":"2024-09-13T12:53:56.141100+0000","last_in_change":"2024-09-13T12:53:38.327038+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T12:51:53.873752+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"23","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"edec8271-f2f4-4f2b-b7b8-710f44c128fa","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6802","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6803","nonce":3230857529}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6804","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6805","nonce":3230857529}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6808","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6809","nonce":3230857529}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6806","nonce":3230857529},{"type":"v1","addr":"172.21.15.42:6807","nonce":3230857529}]},"public_addr":"172.21.15.42:6803/3230857529","cluster_addr":"172.21.15.42:6805/3230857529","heartbeat_back_addr":"172.21.15.42:6809/3230857529","heartbeat_front_addr":"172.21.15.42:6807/3230857529","state":["exists","up"]},{"osd":1,"uuid":"a68de12b-5ed6-474f-9a88-08d119714e95","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6810","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6811","nonce":1160947428}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6812","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6813","nonce":1160947428}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6816","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6817","nonce":1160947428}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6814","nonce":1160947428},{"type":"v1","addr":"172.21.15.42:6815","nonce":1160947428}]},"public_addr":"172.21.15.42:6811/1160947428","cluster_addr":"172.21.15.42:6813/1160947428","heartbeat_back_addr":"172.21.15.42:6817/1160947428","heartbeat_front_addr":"172.21.15.42:6815/1160947428","state":["exists","up"]},{"osd":2,"uuid":"0fa5feee-949c-46a9-8edc-5bc91cbcbe0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6818","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6819","nonce":912984957}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6820","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6821","nonce":912984957}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6824","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6825","nonce":912984957}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6822","nonce":912984957},{"type":"v1","addr":"172.21.15.42:6823","nonce":912984957}]},"public_addr":"172.21.15.42:6819/912984957","cluster_addr":"172.21.15.42:6821/912984957","heartbeat_back_addr":"172.21.15.42:6825/912984957","heartbeat_front_addr":"172.21.15.42:6823/912984957","state":["exists","up"]},{"osd":3,"uuid":"a649835b-3f3c-4831-b752-da0a8a155794","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":27,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6826","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6827","nonce":536044240}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6828","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6829","nonce":536044240}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6832","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6833","nonce":536044240}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.42:6830","nonce":536044240},{"type":"v1","addr":"172.21.15.42:6831","nonce":536044240}]},"public_addr":"172.21.15.42:6827/536044240","cluster_addr":"172.21.15.42:6829/536044240","heartbeat_back_addr":"172.21.15.42:6833/536044240","heartbeat_front_addr":"172.21.15.42:6831/536044240","state":["exists","up"]},{"osd":4,"uuid":"eefb8dfb-dae1-486b-b971-5d54e5ac358c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":32,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6800","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6801","nonce":3781993651}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6802","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6803","nonce":3781993651}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6806","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6807","nonce":3781993651}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6804","nonce":3781993651},{"type":"v1","addr":"172.21.15.46:6805","nonce":3781993651}]},"public_addr":"172.21.15.46:6801/3781993651","cluster_addr":"172.21.15.46:6803/3781993651","heartbeat_back_addr":"172.21.15.46:6807/3781993651","heartbeat_front_addr":"172.21.15.46:6805/3781993651","state":["exists","up"]},{"osd":5,"uuid":"7785b24e-0c12-4283-9572-645442edc6ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":40,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6808","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6809","nonce":2495505474}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6810","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6811","nonce":2495505474}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6814","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6815","nonce":2495505474}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6812","nonce":2495505474},{"type":"v1","addr":"172.21.15.46:6813","nonce":2495505474}]},"public_addr":"172.21.15.46:6809/2495505474","cluster_addr":"172.21.15.46:6811/2495505474","heartbeat_back_addr":"172.21.15.46:6815/2495505474","heartbeat_front_addr":"172.21.15.46:6813/2495505474","state":["exists","up"]},{"osd":6,"uuid":"a977a6a5-0770-47b2-beea-ad49b4c0227f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":51,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6816","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6817","nonce":3075368475}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6818","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6819","nonce":3075368475}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6822","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6823","nonce":3075368475}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6820","nonce":3075368475},{"type":"v1","addr":"172.21.15.46:6821","nonce":3075368475}]},"public_addr":"172.21.15.46:6817/3075368475","cluster_addr":"172.21.15.46:6819/3075368475","heartbeat_back_addr":"172.21.15.46:6823/3075368475","heartbeat_front_addr":"172.21.15.46:6821/3075368475","state":["exists","up"]},{"osd":7,"uuid":"6ba631db-d6a4-4408-92de-9bf900a99dc9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":56,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6824","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6825","nonce":2174137873}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6826","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6827","nonce":2174137873}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6830","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6831","nonce":2174137873}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6828","nonce":2174137873},{"type":"v1","addr":"172.21.15.46:6829","nonce":2174137873}]},"public_addr":"172.21.15.46:6825/2174137873","cluster_addr":"172.21.15.46:6827/2174137873","heartbeat_back_addr":"172.21.15.46:6831/2174137873","heartbeat_front_addr":"172.21.15.46:6829/2174137873","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:50:51.202792+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:51:22.490251+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:51:49.558149+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:52:16.288067+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:52:39.176258+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:03.484085+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:28.136067+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T12:53:54.801101+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.42:0/154408839":"2024-09-14T12:49:34.486008+0000","172.21.15.42:0/1972800177":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6800/3161296684":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6801/273214107":"2024-09-14T12:45:57.248998+0000","172.21.15.42:6800/273214107":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/1533562927":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/2634043947":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/2757003853":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6801/2424236657":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/3321752004":"2024-09-14T12:45:26.746626+0000","172.21.15.42:0/207953028":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/3801247064":"2024-09-14T12:45:57.248998+0000","172.21.15.42:0/2636830803":"2024-09-14T12:45:26.746626+0000","172.21.15.42:6801/3161296684":"2024-09-14T12:49:34.486008+0000","172.21.15.42:6800/2424236657":"2024-09-14T12:45:26.746626+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T12:54:16.373 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:16 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1843442862' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T12:54:16.401 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T12:54:16.402 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T12:54:16.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T12:54:16.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T12:54:16.403 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T12:54:16.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T12:54:16.404 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T12:54:16.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T12:54:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:16 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1843442862' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T12:54:16.832 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.832 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.833 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.835 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.835 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.836 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.841 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:16.841 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:17.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:17 smithi042 ceph-mon[23723]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:17 smithi046 ceph-mon[32268]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:19.445 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:19 smithi042 ceph-mon[23723]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:19 smithi046 ceph-mon[32268]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:21.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:21 smithi042 ceph-mon[23723]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:21.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:21 smithi046 ceph-mon[32268]: pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:22.812 INFO:teuthology.orchestra.run.smithi042.stdout:38654705709 2024-09-13T12:54:22.813 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T12:54:23.343 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:23 smithi042 ceph-mon[23723]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:23 smithi046 ceph-mon[32268]: pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:25.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:25 smithi042 ceph-mon[23723]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:25 smithi046 ceph-mon[32268]: pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:25.993 INFO:teuthology.orchestra.run.smithi042.stdout:240518168585 2024-09-13T12:54:25.994 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T12:54:26.627 INFO:teuthology.orchestra.run.smithi042.stdout:60129542183 2024-09-13T12:54:26.628 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T12:54:27.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:27 smithi042 ceph-mon[23723]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:27.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:27 smithi046 ceph-mon[32268]: pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:29.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:29 smithi042 ceph-mon[23723]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:29 smithi046 ceph-mon[32268]: pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:29.659 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:29.659 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:30.067 INFO:teuthology.orchestra.run.smithi042.stdout:206158430221 2024-09-13T12:54:30.067 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T12:54:30.962 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:30.988 INFO:teuthology.orchestra.run.smithi042.stdout:115964117020 2024-09-13T12:54:30.989 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T12:54:31.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:31 smithi042 ceph-mon[23723]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:31 smithi046 ceph-mon[32268]: pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:33.425 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:33 smithi042 ceph-mon[23723]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:33.455 INFO:teuthology.orchestra.run.smithi042.stdout:171798691860 2024-09-13T12:54:33.455 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T12:54:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:33 smithi046 ceph-mon[32268]: pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:35.324 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:35 smithi042 ceph-mon[23723]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:35.455 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:35 smithi046 ceph-mon[32268]: pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:36.454 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:54:36.454 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:54:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:54:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:54:37.003 INFO:teuthology.orchestra.run.smithi042.stdout:81604378660 2024-09-13T12:54:37.003 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T12:54:37.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:37 smithi042 ceph-mon[23723]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:37.398 INFO:teuthology.orchestra.run.smithi042.stdout:206158430224 2024-09-13T12:54:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:37 smithi046 ceph-mon[32268]: pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:38.125 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:38.340 INFO:teuthology.orchestra.run.smithi042.stdout:240518168588 2024-09-13T12:54:38.340 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:38 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2769499024' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T12:54:38.340 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:38 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3192314589' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T12:54:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:38 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2769499024' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T12:54:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:38 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3192314589' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T12:54:38.963 INFO:tasks.cephadm.ceph_manager.ceph:need seq 240518168585 got 240518168588 for osd.7 2024-09-13T12:54:38.963 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:39.295 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:39.318 INFO:teuthology.orchestra.run.smithi042.stdout:137438953498 2024-09-13T12:54:39.318 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T12:54:39.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:39 smithi042 ceph-mon[23723]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:39.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:39 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3507051073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T12:54:39.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:39 smithi046 ceph-mon[32268]: pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:39.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:39 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3507051073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T12:54:39.938 INFO:teuthology.orchestra.run.smithi042.stdout:38654705712 2024-09-13T12:54:40.146 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:40.525 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705709 got 38654705712 for osd.0 2024-09-13T12:54:40.525 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:41.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:41 smithi042 ceph-mon[23723]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:41.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:41 smithi046 ceph-mon[32268]: pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:42.210 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:42.580 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430221 got 206158430224 for osd.6 2024-09-13T12:54:42.581 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:43.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:43 smithi042 ceph-mon[23723]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:43.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:43 smithi046 ceph-mon[32268]: pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:43.737 INFO:teuthology.orchestra.run.smithi042.stdout:60129542187 2024-09-13T12:54:44.483 INFO:teuthology.orchestra.run.smithi042.stdout:115964117023 2024-09-13T12:54:44.484 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:44 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/733010591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T12:54:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:44 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/733010591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T12:54:45.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:45 smithi042 ceph-mon[23723]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:45.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:45 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3809367638' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T12:54:45.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:45 smithi046 ceph-mon[32268]: pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:45.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:45 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3809367638' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T12:54:45.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542183 got 60129542187 for osd.1 2024-09-13T12:54:45.636 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:46.434 INFO:tasks.cephadm.ceph_manager.ceph:need seq 115964117020 got 115964117023 for osd.3 2024-09-13T12:54:46.434 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:47.123 INFO:teuthology.orchestra.run.smithi042.stdout:171798691863 2024-09-13T12:54:47.336 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:47 smithi042 ceph-mon[23723]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:47.336 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:47 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/533655370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T12:54:47.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:47 smithi046 ceph-mon[32268]: pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:47.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:47 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/533655370' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T12:54:48.189 INFO:teuthology.orchestra.run.smithi042.stdout:81604378662 2024-09-13T12:54:48.391 INFO:tasks.cephadm.ceph_manager.ceph:need seq 171798691860 got 171798691863 for osd.5 2024-09-13T12:54:48.392 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:48.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:48 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/290967593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T12:54:48.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:48 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/290967593' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T12:54:48.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378660 got 81604378662 for osd.2 2024-09-13T12:54:48.892 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:49.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:49 smithi042 ceph-mon[23723]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:49.539 INFO:teuthology.orchestra.run.smithi042.stdout:137438953501 2024-09-13T12:54:49.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:49 smithi046 ceph-mon[32268]: pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:50.191 INFO:tasks.cephadm.ceph_manager.ceph:need seq 137438953498 got 137438953501 for osd.4 2024-09-13T12:54:50.191 DEBUG:teuthology.parallel:result is None 2024-09-13T12:54:50.191 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T12:54:50.191 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T12:54:50.416 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:50.637 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:50 smithi042 ceph-mon[23723]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:50.637 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:50 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/4131118900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T12:54:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:50 smithi046 ceph-mon[32268]: pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:50 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/4131118900' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T12:54:52.148 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:52.152 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T12:54:52.717 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:52 smithi042 ceph-mon[23723]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:52.717 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:52 smithi042 ceph-mon[23723]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:54:52.746 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":206,"stamp":"2024-09-13T12:54:51.416528+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48676,"kb_used_data":6372,"kb_used_omap":0,"kb_used_meta":42240,"kb_avail":749912540,"statfs":{"total":767960285184,"available":767910440960,"internally_reserved":0,"allocated":6524928,"data_stored":3571113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43253760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002025"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-13T12:54:00.169709+0000","last_change":"2024-09-13T12:54:00.169709+0000","last_active":"2024-09-13T12:54:00.169709+0000","last_peered":"2024-09-13T12:54:00.169709+0000","last_clean":"2024-09-13T12:54:00.169709+0000","last_became_active":"2024-09-13T12:54:00.169335+0000","last_became_peered":"2024-09-13T12:54:00.169335+0000","last_unstale":"2024-09-13T12:54:00.169709+0000","last_undegraded":"2024-09-13T12:54:00.169709+0000","last_fullsized":"2024-09-13T12:54:00.169709+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T12:51:53.923880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T12:51:53.923880+0000","last_clean_scrub_stamp":"2024-09-13T12:51:53.923880+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:38.940921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168590,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":776554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73999999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67600000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.37}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6120,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739032,"statfs":{"total":95995035648,"available":95988768768,"internally_reserved":0,"allocated":1146880,"data_stored":776877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.505}]},{"osd":1,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48299999999999998}]},{"osd":2,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.48899999999999999}]},{"osd":3,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.46899999999999997}]},{"osd":4,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52900000000000003}]},{"osd":5,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36899999999999999,"15min":0.35499999999999998},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.5,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.39700000000000002,"15min":0.371},"min":{"1min":0.27600000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.33700000000000002}]},{"osd":2,"last update":"Fri Sep 13 12:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.38800000000000001},"min":{"1min":0.34699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.248},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.41799999999999998,"15min":0.40899999999999997},"min":{"1min":0.29499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.221}]},{"osd":3,"last update":"Fri Sep 13 12:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.65400000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.70599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.32400000000000001}]},{"osd":4,"last update":"Fri Sep 13 12:54:48 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42699999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.85799999999999998,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.439},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65100000000000002,"15min":0.65000000000000002},"min":{"1min":0.42299999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.875,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.432}]},{"osd":5,"last update":"Fri Sep 13 12:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.54700000000000004}]},{"osd":6,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.375}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705715,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 12:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.36499999999999999,"15min":0.33100000000000002},"min":{"1min":0.29399999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.41299999999999998}]},{"osd":2,"last update":"Fri Sep 13 12:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.40300000000000002,"15min":0.39700000000000002},"min":{"1min":0.29099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.39900000000000002,"15min":0.38400000000000001},"min":{"1min":0.309,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.45400000000000001}]},{"osd":3,"last update":"Fri Sep 13 12:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.46300000000000002,"15min":0.45900000000000002},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61799999999999999,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.56299999999999994}]},{"osd":4,"last update":"Fri Sep 13 12:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56899999999999995,"15min":0.56399999999999995},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.59199999999999997,"15min":0.58599999999999997},"min":{"1min":0.47499999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80400000000000005,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.67000000000000004}]},{"osd":5,"last update":"Fri Sep 13 12:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.57699999999999996}]},{"osd":6,"last update":"Fri Sep 13 12:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.498},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.63100000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50900000000000001}]}]},{"osd":2,"up_from":19,"seq":81604378663,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6240,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738912,"statfs":{"total":95995035648,"available":95988645888,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.433,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.33900000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.504}]},{"osd":1,"last update":"Fri Sep 13 12:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.34399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.42499999999999999}]},{"osd":3,"last update":"Fri Sep 13 12:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.47199999999999998,"15min":0.46500000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.47899999999999998}]},{"osd":4,"last update":"Fri Sep 13 12:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.622,"15min":0.61599999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.621,"15min":0.61199999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.81200000000000006}]},{"osd":5,"last update":"Fri Sep 13 12:54:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.877}]},{"osd":6,"last update":"Fri Sep 13 12:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.80200000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.83699999999999997}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.248},{"interface":"front","average":{"1min":0.433,"5min":0.41599999999999998,"15min":0.41299999999999998},"min":{"1min":0.29299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.29899999999999999}]},{"osd":1,"last update":"Fri Sep 13 12:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.42699999999999999,"15min":0.42299999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.42499999999999999,"15min":0.42399999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.32200000000000001}]},{"osd":2,"last update":"Fri Sep 13 12:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.45900000000000002,"15min":0.45800000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.433,"5min":0.45500000000000002,"15min":0.45900000000000002},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.33500000000000002}]},{"osd":4,"last update":"Fri Sep 13 12:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.58799999999999997,"15min":0.57399999999999995},"min":{"1min":0.441,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.56399999999999995}]},{"osd":5,"last update":"Fri Sep 13 12:54:06 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.61599999999999999}]},{"osd":6,"last update":"Fri Sep 13 12:54:30 2024","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":11.728,"5min":11.728,"15min":11.728},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.64700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]},{"osd":4,"up_from":32,"seq":137438953501,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.59099999999999997,"15min":0.60099999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79500000000000004,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.502},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68500000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.52300000000000002}]},{"osd":1,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.59599999999999997,"15min":0.60399999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85199999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79400000000000004,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54600000000000004}]},{"osd":2,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76900000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.69199999999999995},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.73199999999999998,"15min":0.66800000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":11.842000000000001,"5min":11.842000000000001,"15min":11.842000000000001},"last":0.625}]},{"osd":3,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.63300000000000001,"15min":0.64000000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73899999999999999,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.61199999999999999,"15min":0.61499999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82799999999999996,"5min":1.026,"15min":1.026},"last":0.72199999999999998}]},{"osd":5,"last update":"Fri Sep 13 12:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.73499999999999999}]},{"osd":6,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.79200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.84999999999999998}]}]},{"osd":5,"up_from":40,"seq":171798691864,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.35599999999999998}]},{"osd":1,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.308}]},{"osd":2,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.42499999999999999}]},{"osd":3,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.47399999999999998}]},{"osd":4,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.48799999999999999}]},{"osd":6,"last update":"Fri Sep 13 12:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.437},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":0.55900000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T12:54:52.748 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T12:54:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:52 smithi046 ceph-mon[32268]: pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:52 smithi046 ceph-mon[32268]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:54:52.970 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:54.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:54 smithi042 ceph-mon[23723]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:54.696 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:54.697 INFO:teuthology.orchestra.run.smithi042.stderr:dumped all 2024-09-13T12:54:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:54 smithi046 ceph-mon[32268]: pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:55.388 INFO:teuthology.orchestra.run.smithi042.stdout:{"pg_ready":true,"pg_map":{"version":207,"stamp":"2024-09-13T12:54:53.416849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48676,"kb_used_data":6372,"kb_used_omap":0,"kb_used_meta":42240,"kb_avail":749912540,"statfs":{"total":767960285184,"available":767910440960,"internally_reserved":0,"allocated":6524928,"data_stored":3571113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43253760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001991"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":226,"reported_epoch":60,"state":"active+clean","last_fresh":"2024-09-13T12:54:00.169709+0000","last_change":"2024-09-13T12:54:00.169709+0000","last_active":"2024-09-13T12:54:00.169709+0000","last_peered":"2024-09-13T12:54:00.169709+0000","last_clean":"2024-09-13T12:54:00.169709+0000","last_became_active":"2024-09-13T12:54:00.169335+0000","last_became_peered":"2024-09-13T12:54:00.169335+0000","last_unstale":"2024-09-13T12:54:00.169709+0000","last_undegraded":"2024-09-13T12:54:00.169709+0000","last_fullsized":"2024-09-13T12:54:00.169709+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":21,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T12:51:53.923880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T12:51:53.923880+0000","last_clean_scrub_stamp":"2024-09-13T12:51:53.923880+0000","objects_scrubbed":0,"log_size":117,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T21:38:38.940921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3727360,"data_stored":3702496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":56,"seq":240518168591,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6052,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":4928,"kb_avail":93739100,"statfs":{"total":95995035648,"available":95988838400,"internally_reserved":0,"allocated":1142784,"data_stored":776554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5046272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.48799999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40799999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58099999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55600000000000005}]}]},{"osd":6,"up_from":48,"seq":206158430227,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6120,"kb_used_data":1120,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739032,"statfs":{"total":95995035648,"available":95988768768,"internally_reserved":0,"allocated":1146880,"data_stored":776877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.505}]},{"osd":1,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.48299999999999998}]},{"osd":2,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.48899999999999999}]},{"osd":3,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.46899999999999997}]},{"osd":4,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52900000000000003}]},{"osd":5,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542189,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6308,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36899999999999999,"15min":0.35499999999999998},"min":{"1min":0.26500000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.5,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.39700000000000002,"15min":0.371},"min":{"1min":0.27600000000000002,"5min":0.221,"15min":0.221},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.33700000000000002}]},{"osd":2,"last update":"Fri Sep 13 12:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.38800000000000001},"min":{"1min":0.34699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.248},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.41799999999999998,"15min":0.40899999999999997},"min":{"1min":0.29499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.221}]},{"osd":3,"last update":"Fri Sep 13 12:54:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.65400000000000003,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.70599999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.32400000000000001}]},{"osd":4,"last update":"Fri Sep 13 12:54:48 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.42699999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.85799999999999998,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.439},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65100000000000002,"15min":0.65000000000000002},"min":{"1min":0.42299999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.875,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.432}]},{"osd":5,"last update":"Fri Sep 13 12:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.54700000000000004}]},{"osd":6,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"min":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.375}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]}]},{"osd":0,"up_from":9,"seq":38654705716,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6892,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738260,"statfs":{"total":95995035648,"available":95987978240,"internally_reserved":0,"allocated":1150976,"data_stored":777142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 12:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.35699999999999998,"15min":0.32900000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.36499999999999999,"15min":0.33100000000000002},"min":{"1min":0.29399999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.42399999999999999}]},{"osd":2,"last update":"Fri Sep 13 12:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.40300000000000002,"15min":0.39700000000000002},"min":{"1min":0.29099999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.437},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.39900000000000002,"15min":0.38400000000000001},"min":{"1min":0.309,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.48299999999999998}]},{"osd":3,"last update":"Fri Sep 13 12:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.42499999999999999,"15min":0.42299999999999999},"min":{"1min":0.309,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.46300000000000002,"15min":0.45900000000000002},"min":{"1min":0.35399999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61799999999999999,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.54800000000000004}]},{"osd":4,"last update":"Fri Sep 13 12:54:43 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56899999999999995,"15min":0.56399999999999995},"min":{"1min":0.371,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.59199999999999997,"15min":0.58599999999999997},"min":{"1min":0.47499999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.80400000000000005,"5min":0.88700000000000001,"15min":0.88700000000000001},"last":0.874}]},{"osd":5,"last update":"Fri Sep 13 12:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.90900000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.83799999999999997}]},{"osd":6,"last update":"Fri Sep 13 12:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.751}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77500000000000002}]}]},{"osd":2,"up_from":19,"seq":81604378664,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6240,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":93738912,"statfs":{"total":95995035648,"available":95988645888,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.43099999999999999,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.28799999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.81699999999999995,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.379},{"interface":"front","average":{"1min":0.433,"5min":0.38300000000000001,"15min":0.375},"min":{"1min":0.33900000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.36499999999999999}]},{"osd":1,"last update":"Fri Sep 13 12:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.41599999999999998,"15min":0.40600000000000003},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.39300000000000002},"min":{"1min":0.34399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.308}]},{"osd":3,"last update":"Fri Sep 13 12:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.47199999999999998,"15min":0.46500000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.232,"5min":1.232,"15min":1.232},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.34499999999999997}]},{"osd":4,"last update":"Fri Sep 13 12:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.622,"15min":0.61599999999999999},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.495},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.621,"15min":0.61199999999999999},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.57599999999999996}]},{"osd":5,"last update":"Fri Sep 13 12:54:06 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.55200000000000005}]},{"osd":6,"last update":"Fri Sep 13 12:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.1559999999999999,"5min":1.1559999999999999,"15min":1.1559999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.54000000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56299999999999994}]}]},{"osd":3,"up_from":27,"seq":115964117025,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5664,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739488,"statfs":{"total":95995035648,"available":95989235712,"internally_reserved":0,"allocated":614400,"data_stored":247949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.248},{"interface":"front","average":{"1min":0.433,"5min":0.41599999999999998,"15min":0.41299999999999998},"min":{"1min":0.29299999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.29899999999999999}]},{"osd":1,"last update":"Fri Sep 13 12:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.42699999999999999,"15min":0.42299999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.42499999999999999,"15min":0.42399999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.32200000000000001}]},{"osd":2,"last update":"Fri Sep 13 12:54:23 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.45900000000000002,"15min":0.45800000000000002},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.433,"5min":0.45500000000000002,"15min":0.45900000000000002},"min":{"1min":0.32200000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.33500000000000002}]},{"osd":4,"last update":"Fri Sep 13 12:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.56499999999999995,"15min":0.55500000000000005},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.58799999999999997,"15min":0.57399999999999995},"min":{"1min":0.441,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.56399999999999995}]},{"osd":5,"last update":"Fri Sep 13 12:54:06 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.61599999999999999}]},{"osd":6,"last update":"Fri Sep 13 12:54:30 2024","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.1599999999999999,"15min":1.1599999999999999},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":11.728,"5min":11.728,"15min":11.728},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.64700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]}]},{"osd":4,"up_from":32,"seq":137438953502,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5732,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739420,"statfs":{"total":95995035648,"available":95989166080,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.59099999999999997,"15min":0.60099999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79500000000000004,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.56299999999999994,"15min":0.56699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68500000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.57299999999999995}]},{"osd":1,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.59599999999999997,"15min":0.60399999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85199999999999998,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.58799999999999997,"15min":0.58999999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.79400000000000004,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.54800000000000004}]},{"osd":2,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.59399999999999997,"15min":0.59999999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76900000000000002,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.73199999999999998,"15min":0.66800000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":11.842000000000001,"5min":11.842000000000001,"15min":11.842000000000001},"last":0.52600000000000002}]},{"osd":3,"last update":"Fri Sep 13 12:54:42 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.63300000000000001,"15min":0.64000000000000001},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73899999999999999,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.61199999999999999,"15min":0.61499999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.82799999999999996,"5min":1.026,"15min":1.026},"last":0.51100000000000001}]},{"osd":5,"last update":"Fri Sep 13 12:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.437,"15min":0.437},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.379}]},{"osd":6,"last update":"Fri Sep 13 12:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.39900000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59999999999999998}]}]},{"osd":5,"up_from":40,"seq":171798691865,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5668,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739484,"statfs":{"total":95995035648,"available":95989231616,"internally_reserved":0,"allocated":618496,"data_stored":248214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.746}]},{"osd":1,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.68400000000000005}]},{"osd":2,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.71599999999999997}]},{"osd":3,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.81399999999999995}]},{"osd":4,"last update":"Fri Sep 13 12:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.052,"5min":1.052,"15min":1.052},"last":0.55500000000000005}]},{"osd":6,"last update":"Fri Sep 13 12:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.105,"5min":1.105,"15min":1.105},"last":0.65300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T12:54:55.389 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T12:54:55.390 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T12:54:55.390 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T12:54:55.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T12:54:55.612 INFO:teuthology.orchestra.run.smithi042.stderr:Inferring config /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/mon.smithi042/config 2024-09-13T12:54:55.730 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:55 smithi042 ceph-mon[23723]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:54:55.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:55 smithi046 ceph-mon[32268]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:54:56.678 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:56 smithi042 ceph-mon[23723]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:56 smithi046 ceph-mon[32268]: pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:57.363 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:54:57.363 INFO:teuthology.orchestra.run.smithi042.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T12:54:57.684 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:57 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2974305166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T12:54:57.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:57 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2974305166' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T12:54:58.001 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T12:54:58.002 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T12:54:58.002 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T12:54:58.018 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:54:58.018 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph orch status' 2024-09-13T12:54:58.617 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:58 smithi042 ceph-mon[23723]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:58 smithi046 ceph-mon[32268]: pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:54:59.474 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:54:59.474 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:54:59.474 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:54:59.474 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:54:59 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:54:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:54:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:54:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:54:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:54:59 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:54:59.869 INFO:teuthology.orchestra.run.smithi042.stdout:Backend: cephadm 2024-09-13T12:54:59.869 INFO:teuthology.orchestra.run.smithi042.stdout:Available: Yes 2024-09-13T12:54:59.869 INFO:teuthology.orchestra.run.smithi042.stdout:Paused: No 2024-09-13T12:55:00.507 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ps' 2024-09-13T12:55:01.093 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:00 smithi042 ceph-mon[23723]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:01.093 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:00 smithi042 ceph-mon[23723]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:01.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:00 smithi046 ceph-mon[32268]: pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:01.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:00 smithi046 ceph-mon[32268]: from='client.14560 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:02.523 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:55:02.523 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (4m) 2m ago 8m 15.5M - ba2b418f427c 9e454cd0ade6 2024-09-13T12:55:02.523 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (8m) 2m ago 8m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (5m) 68s ago 5m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (4m) 2m ago 5m 44.5M - 8.3.5 dad864ee21e9 ca253f1b9c19 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (9m) 2m ago 9m 448M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (5m) 68s ago 5m 410M - 17.2.0 e1d6a67b021e 278d4d510a6d 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (9m) 2m ago 10m 44.9M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (5m) 68s ago 5m 34.7M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (5m) 2m ago 5m 18.2M - 1dbe0e931976 5a333e64c22b 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (4m) 68s ago 4m 18.0M - 1dbe0e931976 7086f6b8e52b 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (4m) 2m ago 4m 33.4M 3217M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (3m) 2m ago 3m 34.2M 3217M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (3m) 2m ago 3m 31.0M 3217M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (2m) 2m ago 2m 10.0M 3217M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:55:02.524 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (2m) 68s ago 2m 33.4M 3985M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:55:02.525 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (2m) 68s ago 2m 35.9M 3985M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:55:02.525 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (98s) 68s ago 98s 32.5M 3985M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:55:02.525 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (72s) 68s ago 71s 10.8M 3985M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:55:02.525 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (4m) 2m ago 4m 54.2M - 514e6a882f6e 147d67f2f5fb 2024-09-13T12:55:03.128 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:02 smithi042 ceph-mon[23723]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:03.156 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph orch ls' 2024-09-13T12:55:03.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:02 smithi046 ceph-mon[32268]: pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:04.126 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:03 smithi042 ceph-mon[23723]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:04.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:03 smithi046 ceph-mon[32268]: from='client.14564 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager ?:9093,9094 1/1 2m ago 9m count:1 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:crash 2/2 2m ago 9m * 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:grafana ?:3000 1/1 2m ago 9m count:1 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:mgr 2/2 2m ago 9m count:2 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:mon 2/2 2m ago 8m smithi042:172.21.15.42=smithi042;smithi046:172.21.15.46=smithi046;count:2 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter ?:9100 2/2 2m ago 9m * 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:osd 8 2m ago - 2024-09-13T12:55:05.078 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus ?:9095 1/1 2m ago 9m count:1 2024-09-13T12:55:05.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:04 smithi042 ceph-mon[23723]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:05.268 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:04 smithi046 ceph-mon[32268]: pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:05.832 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph orch host ls' 2024-09-13T12:55:06.139 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:05 smithi042 ceph-mon[23723]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:06.286 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:05 smithi046 ceph-mon[32268]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:06 smithi046 ceph-mon[32268]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:07.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:06 smithi042 ceph-mon[23723]: pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:07.844 INFO:teuthology.orchestra.run.smithi042.stdout:HOST ADDR LABELS STATUS 2024-09-13T12:55:07.844 INFO:teuthology.orchestra.run.smithi042.stdout:smithi042 172.21.15.42 2024-09-13T12:55:07.844 INFO:teuthology.orchestra.run.smithi042.stdout:smithi046 172.21.15.46 2024-09-13T12:55:07.844 INFO:teuthology.orchestra.run.smithi042.stdout:2 hosts in cluster 2024-09-13T12:55:08.619 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph orch device ls' 2024-09-13T12:55:09.185 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:08 smithi042 ceph-mon[23723]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:09.185 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:08 smithi042 ceph-mon[23723]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:08 smithi046 ceph-mon[32268]: pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:08 smithi046 ceph-mon[32268]: from='client.14572 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:10.601 INFO:teuthology.orchestra.run.smithi042.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REJECT REASONS 2024-09-13T12:55:10.601 INFO:teuthology.orchestra.run.smithi042.stdout:smithi042 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB75130021480DGN 480G LVM detected, locked 2024-09-13T12:55:10.602 INFO:teuthology.orchestra.run.smithi042.stdout:smithi046 /dev/nvme0n1 ssd INTEL_SSDPED1D480GA_PHMB7512009A480DGN 480G LVM detected, locked 2024-09-13T12:55:11.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:10 smithi042 ceph-mon[23723]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:11.322 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T12:55:11.335 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:55:11.336 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T12:55:11.366 INFO:teuthology.orchestra.run.smithi042.stderr:+ systemctl stop nfs-server 2024-09-13T12:55:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:10 smithi046 ceph-mon[32268]: pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:11.374 INFO:tasks.vip:Running commands on role host.b host ubuntu@smithi046.front.sepia.ceph.com 2024-09-13T12:55:11.375 DEBUG:teuthology.orchestra.run.smithi046:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'systemctl stop nfs-server' 2024-09-13T12:55:11.408 INFO:teuthology.orchestra.run.smithi046.stderr:+ systemctl stop nfs-server 2024-09-13T12:55:11.417 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T12:55:11.425 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:55:11.426 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foofs' 2024-09-13T12:55:11.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:11 smithi042 ceph-mon[23723]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:11 smithi046 ceph-mon[32268]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:12.967 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:12 smithi042 ceph-mon[23723]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:12 smithi046 ceph-mon[32268]: pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:14.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:13 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T12:55:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:13 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]: dispatch 2024-09-13T12:55:15.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:14 smithi042 ceph-mon[23723]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:15.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:14 smithi042 ceph-mon[23723]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:15.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:14 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T12:55:15.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:14 smithi042 ceph-mon[23723]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T12:55:15.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:14 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T12:55:15.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:14 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[23718]: 2024-09-13T12:55:14.956+0000 7f66a2a27700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T12:55:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:14 smithi046 ceph-mon[32268]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foofs", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:14 smithi046 ceph-mon[32268]: pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:14 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.meta"}]': finished 2024-09-13T12:55:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:14 smithi046 ceph-mon[32268]: osdmap e61: 8 total, 8 up, 8 in 2024-09-13T12:55:15.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:14 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]: dispatch 2024-09-13T12:55:15.746 INFO:teuthology.run_tasks:Running task cephadm.wait_for_service... 2024-09-13T12:55:15.755 INFO:tasks.cephadm:Waiting for ceph service mds.foofs to start (timeout 300)... 2024-09-13T12:55:15.755 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T12:55:15.978 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: fsmap foofs:0 2024-09-13T12:55:15.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: Saving service mds.foofs spec with placement count:2 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:15.980 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:15 smithi042 ceph-mon[23723]: Deploying daemon mds.foofs.smithi042.hljzvq on smithi042 2024-09-13T12:55:16.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foofs.data"}]': finished 2024-09-13T12:55:16.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: osdmap e62: 8 total, 8 up, 8 in 2024-09-13T12:55:16.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]: dispatch 2024-09-13T12:55:16.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: osdmap e63: 8 total, 8 up, 8 in 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "fs new", "fs_name": "foofs", "metadata": "cephfs.foofs.meta", "data": "cephfs.foofs.data"}]': finished 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: fsmap foofs:0 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: Saving service mds.foofs spec with placement count:2 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T12:55:16.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T12:55:16.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:16.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:15 smithi046 ceph-mon[32268]: Deploying daemon mds.foofs.smithi042.hljzvq on smithi042 2024-09-13T12:55:17.235 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:16 smithi042 ceph-mon[23723]: pgmap v221: 65 pgs: 8 creating+peering, 12 active+clean, 45 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:17.235 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:16 smithi042 ceph-mon[23723]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T12:55:17.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:16 smithi046 ceph-mon[32268]: pgmap v221: 65 pgs: 8 creating+peering, 12 active+clean, 45 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:17.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:16 smithi046 ceph-mon[32268]: osdmap e64: 8 total, 8 up, 8 in 2024-09-13T12:55:18.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:17 smithi042 ceph-mon[23723]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T12:55:18.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:17 smithi046 ceph-mon[32268]: osdmap e65: 8 total, 8 up, 8 in 2024-09-13T12:55:19.019 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:55:19.019 INFO:teuthology.orchestra.run.smithi042.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T12:45:46.197476Z", "last_refresh": "2024-09-13T12:52:19.042590Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T12:45:40.455782Z", "last_refresh": "2024-09-13T12:52:19.042658Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T12:45:43.585390Z", "last_refresh": "2024-09-13T12:52:19.042722Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T12:55:14.976701Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T12:55:14.969249Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T12:45:38.212274Z", "last_refresh": "2024-09-13T12:52:19.042510Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi042:172.21.15.42=smithi042", "smithi046:172.21.15.46=smithi046"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T12:46:27.928075Z", "last_refresh": "2024-09-13T12:52:19.042372Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T12:45:44.804740Z", "last_refresh": "2024-09-13T12:52:19.042785Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T12:52:19.042936Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T12:45:42.338062Z", "last_refresh": "2024-09-13T12:52:19.042871Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T12:55:19.225 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:18 smithi042 ceph-mon[23723]: pgmap v224: 65 pgs: 8 creating+peering, 39 active+clean, 18 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:18 smithi046 ceph-mon[32268]: pgmap v224: 65 pgs: 8 creating+peering, 39 active+clean, 18 unknown; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:19.837 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T12:55:20.074 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:19 smithi042 ceph-mon[23723]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:20.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:19 smithi046 ceph-mon[32268]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:20.838 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T12:55:21.167 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: pgmap v225: 65 pgs: 5 creating+peering, 60 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:21.167 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: Deploying daemon mds.foofs.smithi046.nqnolk on smithi046 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: mds.? [v2:172.21.15.42:6834/54759589,v1:172.21.15.42:6835/54759589] up:boot 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: daemon mds.foofs.smithi042.hljzvq assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: Cluster is now healthy 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: fsmap foofs:0 1 up:standby 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"}]: dispatch 2024-09-13T12:55:21.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:creating} 2024-09-13T12:55:21.169 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:21 smithi042 ceph-mon[23723]: daemon mds.foofs.smithi042.hljzvq is now active in filesystem foofs as rank 0 2024-09-13T12:55:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: pgmap v225: 65 pgs: 5 creating+peering, 60 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: Deploying daemon mds.foofs.smithi046.nqnolk on smithi046 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: mds.? [v2:172.21.15.42:6834/54759589,v1:172.21.15.42:6835/54759589] up:boot 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: daemon mds.foofs.smithi042.hljzvq assigned to filesystem foofs as rank 0 (now has 1 ranks) 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: Cluster is now healthy 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: fsmap foofs:0 1 up:standby 2024-09-13T12:55:21.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"}]: dispatch 2024-09-13T12:55:21.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:creating} 2024-09-13T12:55:21.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:21 smithi046 ceph-mon[32268]: daemon mds.foofs.smithi042.hljzvq is now active in filesystem foofs as rank 0 2024-09-13T12:55:22.739 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:55:22.740 INFO:teuthology.orchestra.run.smithi042.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T12:45:46.197476Z", "last_refresh": "2024-09-13T12:52:19.042590Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T12:45:40.455782Z", "last_refresh": "2024-09-13T12:52:19.042658Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T12:45:43.585390Z", "last_refresh": "2024-09-13T12:52:19.042722Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T12:55:14.976701Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T12:55:14.969249Z", "running": 0, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T12:45:38.212274Z", "last_refresh": "2024-09-13T12:52:19.042510Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi042:172.21.15.42=smithi042", "smithi046:172.21.15.46=smithi046"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T12:46:27.928075Z", "last_refresh": "2024-09-13T12:52:19.042372Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T12:45:44.804740Z", "last_refresh": "2024-09-13T12:52:19.042785Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T12:52:19.042936Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T12:45:42.338062Z", "last_refresh": "2024-09-13T12:52:19.042871Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T12:55:23.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: pgmap v226: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: mds.? [v2:172.21.15.42:6834/54759589,v1:172.21.15.42:6835/54759589] up:active 2024-09-13T12:55:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 2024-09-13T12:55:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:22 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: pgmap v226: 65 pgs: 65 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: mds.? [v2:172.21.15.42:6834/54759589,v1:172.21.15.42:6835/54759589] up:active 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:23.246 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:22 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:23.424 INFO:tasks.cephadm:mds.foofs has 0/2 2024-09-13T12:55:24.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:24 smithi046 ceph-mon[32268]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:24.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:24 smithi046 ceph-mon[32268]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:boot 2024-09-13T12:55:24.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:24 smithi046 ceph-mon[32268]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T12:55:24.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:24 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"}]: dispatch 2024-09-13T12:55:24.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:24 smithi046 ceph-mon[32268]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T12:55:24.426 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T12:55:24.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:24 smithi042 ceph-mon[23723]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:24.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:24 smithi042 ceph-mon[23723]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:boot 2024-09-13T12:55:24.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:24 smithi042 ceph-mon[23723]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T12:55:24.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:24 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"}]: dispatch 2024-09-13T12:55:24.450 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:24 smithi042 ceph-mon[23723]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T12:55:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:25 smithi046 ceph-mon[32268]: pgmap v227: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-09-13T12:55:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:25 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:25.412 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:25 smithi042 ceph-mon[23723]: pgmap v227: 65 pgs: 65 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1023 B/s wr, 3 op/s 2024-09-13T12:55:25.413 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:25 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:26.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:26 smithi042 ceph-mon[23723]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T12:55:26.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:26 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:26.791 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:55:26.792 INFO:teuthology.orchestra.run.smithi042.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T12:45:46.197476Z", "last_refresh": "2024-09-13T12:52:19.042590Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T12:45:40.455782Z", "last_refresh": "2024-09-13T12:52:19.042658Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T12:45:43.585390Z", "last_refresh": "2024-09-13T12:52:19.042722Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T12:55:14.976701Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T12:55:14.969249Z", "last_refresh": "2024-09-13T12:55:24.530832Z", "running": 1, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T12:45:38.212274Z", "last_refresh": "2024-09-13T12:52:19.042510Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi042:172.21.15.42=smithi042", "smithi046:172.21.15.46=smithi046"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T12:46:27.928075Z", "last_refresh": "2024-09-13T12:52:19.042372Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T12:45:44.804740Z", "last_refresh": "2024-09-13T12:52:19.042785Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T12:52:19.042936Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T12:45:42.338062Z", "last_refresh": "2024-09-13T12:52:19.042871Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T12:55:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:26 smithi046 ceph-mon[32268]: pgmap v228: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2024-09-13T12:55:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:26 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:27.562 INFO:tasks.cephadm:mds.foofs has 1/2 2024-09-13T12:55:27.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:27 smithi042 ceph-mon[23723]: from='client.14600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:27.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:27 smithi046 ceph-mon[32268]: from='client.14600 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:28.544 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:28 smithi042 ceph-mon[23723]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-13T12:55:28.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T12:55:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:28 smithi046 ceph-mon[32268]: pgmap v229: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2024-09-13T12:55:30.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:30 smithi042 ceph-mon[23723]: pgmap v230: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T12:55:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:30 smithi046 ceph-mon[32268]: pgmap v230: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T12:55:31.147 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:55:31.147 INFO:teuthology.orchestra.run.smithi042.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T12:45:46.197476Z", "last_refresh": "2024-09-13T12:55:31.021503Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T12:45:40.455782Z", "last_refresh": "2024-09-13T12:55:24.529782Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "status": {"created": "2024-09-13T12:45:43.585390Z", "last_refresh": "2024-09-13T12:55:31.021636Z", "ports": [3000], "running": 1, "size": 1}}, {"events": ["2024-09-13T12:55:14.976701Z service:mds.foofs [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T12:55:14.969249Z", "last_refresh": "2024-09-13T12:55:24.530832Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T12:45:38.212274Z", "last_refresh": "2024-09-13T12:55:24.529988Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi042:172.21.15.42=smithi042", "smithi046:172.21.15.46=smithi046"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T12:46:27.928075Z", "last_refresh": "2024-09-13T12:55:24.530118Z", "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T12:45:44.804740Z", "last_refresh": "2024-09-13T12:55:24.530233Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", "container_image_name": "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", "last_refresh": "2024-09-13T12:55:24.530357Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T12:45:42.338062Z", "last_refresh": "2024-09-13T12:55:31.021762Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T12:55:31.870 INFO:tasks.cephadm:mds.foofs has 2/2 2024-09-13T12:55:31.870 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T12:55:31.882 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:55:31.882 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2' 2024-09-13T12:55:32.195 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:32.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:32.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:32.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:32.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:32.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:32.196 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:32 smithi042 ceph-mon[23723]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:32.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:32 smithi046 ceph-mon[32268]: from='client.14604 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T12:55:33.223 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:33 smithi042 ceph-mon[23723]: pgmap v231: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T12:55:33.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:33 smithi046 ceph-mon[32268]: pgmap v231: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T12:55:34.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:34 smithi042 ceph-mon[23723]: pgmap v232: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T12:55:34.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:34 smithi042 ceph-mon[23723]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:34.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:34 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T12:55:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:34 smithi046 ceph-mon[32268]: pgmap v232: 65 pgs: 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-13T12:55:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:34 smithi046 ceph-mon[32268]: from='client.14608 -' entity='client.admin' cmd=[{"prefix": "nfs cluster create", "cluster_id": "foo", "placement": "2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:34 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool create", "pool": ".nfs"}]: dispatch 2024-09-13T12:55:35.471 INFO:teuthology.orchestra.run.smithi042.stdout:NFS Cluster Created Successfully 2024-09-13T12:55:35.632 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T12:55:35.632 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:35 smithi046 ceph-mon[32268]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T12:55:35.632 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T12:55:35.632 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:35 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:35.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool create", "pool": ".nfs"}]': finished 2024-09-13T12:55:35.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:35 smithi042 ceph-mon[23723]: osdmap e66: 8 total, 8 up, 8 in 2024-09-13T12:55:35.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]: dispatch 2024-09-13T12:55:35.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:35 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.152 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake' 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: pgmap v235: 97 pgs: 8 creating+peering, 24 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: Saving service nfs.foo spec with placement count:2 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:36.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T12:55:36.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pool application enable", "pool": ".nfs", "app": "nfs"}]': finished 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: osdmap e67: 8 total, 8 up, 8 in 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: pgmap v235: 97 pgs: 8 creating+peering, 24 unknown, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: Saving service nfs.foo spec with placement count:2 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T12:55:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:37.722 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:37 smithi042 ceph-mon[23723]: Rados config object exists: conf-nfs.foo 2024-09-13T12:55:37.723 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:37 smithi042 ceph-mon[23723]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg-rgw 2024-09-13T12:55:37.723 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:37 smithi042 ceph-mon[23723]: Deploying daemon nfs.foo.0.0.smithi042.fnqkqg on smithi042 2024-09-13T12:55:37.723 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:37 smithi042 ceph-mon[23723]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T12:55:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:37 smithi046 ceph-mon[32268]: Rados config object exists: conf-nfs.foo 2024-09-13T12:55:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:37 smithi046 ceph-mon[32268]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg-rgw 2024-09-13T12:55:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:37 smithi046 ceph-mon[32268]: Deploying daemon nfs.foo.0.0.smithi042.fnqkqg on smithi042 2024-09-13T12:55:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:37 smithi046 ceph-mon[32268]: osdmap e68: 8 total, 8 up, 8 in 2024-09-13T12:55:38.307 INFO:teuthology.orchestra.run.smithi042.stderr:Invalid command: Unexpected argument '--clusterid' 2024-09-13T12:55:38.308 INFO:teuthology.orchestra.run.smithi042.stderr:nfs export create cephfs [] [--readonly] [--client_addr ...] [--squash ] : Create a CephFS export 2024-09-13T12:55:38.308 INFO:teuthology.orchestra.run.smithi042.stderr:Error EINVAL: invalid command 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout: "bind": "/fake", 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout: "fs": "foofs", 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout: "path": "/", 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout: "cluster": "foo", 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout: "mode": "RW" 2024-09-13T12:55:38.657 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:55:38.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:38 smithi042 ceph-mon[23723]: pgmap v237: 97 pgs: 8 creating+peering, 13 unknown, 76 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:38.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:38 smithi042 ceph-mon[23723]: mgrmap e20: smithi042.jhsfyp(active, since 6m), standbys: smithi046.hcgddm 2024-09-13T12:55:38.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:38 smithi046 ceph-mon[32268]: pgmap v237: 97 pgs: 8 creating+peering, 13 unknown, 76 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2024-09-13T12:55:38.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:38 smithi046 ceph-mon[32268]: mgrmap e20: smithi042.jhsfyp(active, since 6m), standbys: smithi046.hcgddm 2024-09-13T12:55:39.390 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done' 2024-09-13T12:55:39.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:39 smithi042 ceph-mon[23723]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:39.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T12:55:39.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T12:55:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:39 smithi046 ceph-mon[32268]: from='client.14630 -' entity='client.admin' cmd=[{"prefix": "nfs export create cephfs", "fsname": "foofs", "cluster_id": "foo", "pseudo_path": "/fake", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]: dispatch 2024-09-13T12:55:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo, allow rw tag cephfs data=foofs", "mds", "allow rw path=/"], "format": "json"}]': finished 2024-09-13T12:55:40.971 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:40 smithi042 ceph-mon[23723]: pgmap v238: 97 pgs: 8 creating+peering, 89 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-09-13T12:55:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:40 smithi046 ceph-mon[32268]: pgmap v238: 97 pgs: 8 creating+peering, 89 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 511 B/s wr, 0 op/s 2024-09-13T12:55:43.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: pgmap v239: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 146 B/s rd, 584 B/s wr, 0 op/s 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T12:55:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:42 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: pgmap v239: 97 pgs: 97 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 146 B/s rd, 584 B/s wr, 0 op/s 2024-09-13T12:55:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb 2024-09-13T12:55:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]: dispatch 2024-09-13T12:55:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]}]': finished 2024-09-13T12:55:43.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T12:55:43.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]: dispatch 2024-09-13T12:55:43.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T12:55:43.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:42 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:44.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:43 smithi042 ceph-mon[23723]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:44.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:43 smithi046 ceph-mon[32268]: from='client.14650 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:44 smithi042 ceph-mon[23723]: pgmap v240: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 511 B/s wr, 0 op/s 2024-09-13T12:55:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:44 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T12:55:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:44 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T12:55:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:44 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T12:55:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:44 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T12:55:45.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:44 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:45.324 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:44 smithi046 ceph-mon[32268]: pgmap v240: 97 pgs: 97 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 511 B/s wr, 0 op/s 2024-09-13T12:55:45.324 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:44 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]: dispatch 2024-09-13T12:55:45.324 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:44 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T12:55:45.325 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:44 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-09-13T12:55:45.325 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:44 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-09-13T12:55:45.325 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:44 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:46.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:45 smithi042 ceph-mon[23723]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:46.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:45 smithi042 ceph-mon[23723]: Rados config object exists: conf-nfs.foo 2024-09-13T12:55:46.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:45 smithi042 ceph-mon[23723]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb-rgw 2024-09-13T12:55:46.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:45 smithi042 ceph-mon[23723]: Deploying daemon nfs.foo.1.0.smithi046.jjnsrb on smithi046 2024-09-13T12:55:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:45 smithi046 ceph-mon[32268]: from='client.14654 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:45 smithi046 ceph-mon[32268]: Rados config object exists: conf-nfs.foo 2024-09-13T12:55:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:45 smithi046 ceph-mon[32268]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb-rgw 2024-09-13T12:55:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:45 smithi046 ceph-mon[32268]: Deploying daemon nfs.foo.1.0.smithi046.jjnsrb on smithi046 2024-09-13T12:55:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:46 smithi046 ceph-mon[32268]: pgmap v241: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-13T12:55:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:46 smithi046 ceph-mon[32268]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:47.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:46 smithi042 ceph-mon[23723]: pgmap v241: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-09-13T12:55:47.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:46 smithi042 ceph-mon[23723]: from='client.14668 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:48.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:47 smithi042 ceph-mon[23723]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:48.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:47 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:48.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:47 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:48.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:47 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:48.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:47 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:48.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:47 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:47 smithi046 ceph-mon[32268]: from='client.14672 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:47 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:47 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:47 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:47 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:48.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:47 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:49.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:48 smithi042 ceph-mon[23723]: pgmap v242: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T12:55:49.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:48 smithi042 ceph-mon[23723]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:49.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:48 smithi046 ceph-mon[32268]: pgmap v242: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.4 KiB/s wr, 2 op/s 2024-09-13T12:55:49.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:48 smithi046 ceph-mon[32268]: from='client.14686 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:51.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:50 smithi042 ceph-mon[23723]: pgmap v243: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-09-13T12:55:51.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:50 smithi042 ceph-mon[23723]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:51.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:50 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:51.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:50 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:51.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:50 smithi046 ceph-mon[32268]: pgmap v243: 97 pgs: 97 active+clean; 580 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-09-13T12:55:51.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:50 smithi046 ceph-mon[32268]: from='client.14690 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:51.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:50 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:51.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:50 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:52.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:51 smithi042 ceph-mon[23723]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:52.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:51 smithi046 ceph-mon[32268]: from='client.14694 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:53.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: pgmap v244: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-09-13T12:55:53.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:53.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: pgmap v244: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='client.14698 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:55:53.578 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo ?:2049 2/2 3s ago 18s count:2 2024-09-13T12:55:54.373 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T12:55:54.383 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:55:54.383 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mkdir /mnt/foo' 2024-09-13T12:55:54.412 INFO:teuthology.orchestra.run.smithi042.stderr:+ mkdir /mnt/foo 2024-09-13T12:55:54.431 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done' 2024-09-13T12:55:54.498 INFO:teuthology.orchestra.run.smithi042.stderr:++ hostname 2024-09-13T12:55:54.498 INFO:teuthology.orchestra.run.smithi042.stderr:+ mount -t nfs smithi042:/fake /mnt/foo -o sync 2024-09-13T12:55:54.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'echo test > /mnt/foo/testfile' 2024-09-13T12:55:55.014 INFO:teuthology.orchestra.run.smithi042.stderr:+ echo test 2024-09-13T12:55:55.059 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c sync 2024-09-13T12:55:55.122 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:55 smithi042 ceph-mon[23723]: pgmap v245: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-09-13T12:55:55.122 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:55 smithi042 ceph-mon[23723]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:55.125 INFO:teuthology.orchestra.run.smithi042.stderr:+ sync 2024-09-13T12:55:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:55 smithi046 ceph-mon[32268]: pgmap v245: 97 pgs: 97 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-09-13T12:55:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:55 smithi046 ceph-mon[32268]: from='client.14702 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:55:55.386 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T12:55:55.397 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T12:55:55.397 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T12:55:55.398 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T12:55:55.398 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:55:55.399 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T12:55:55.399 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T12:55:55.399 INFO:teuthology.task.sequential:In sequential, running task exec... 2024-09-13T12:55:55.400 INFO:teuthology.task.exec:Executing custom commands... 2024-09-13T12:55:55.400 INFO:teuthology.task.exec:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:55:55.400 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 600 || true' 2024-09-13T12:55:55.461 INFO:teuthology.orchestra.run.smithi042.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T12:55:55.462 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T12:55:55.462 INFO:teuthology.orchestra.run.smithi042.stdout:Running for 600 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 120 secs 2024-09-13T12:55:55.463 INFO:teuthology.orchestra.run.smithi042.stdout:4 of 5 processes prepared for launch 0 sec 2024-09-13T12:55:55.563 INFO:teuthology.orchestra.run.smithi042.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T12:55:55.564 INFO:teuthology.orchestra.run.smithi042.stdout:releasing clients 2024-09-13T12:55:56.563 INFO:teuthology.orchestra.run.smithi042.stdout: 5 297 54.82 MB/sec warmup 1 sec latency 10.421 ms 2024-09-13T12:55:57.207 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:57 smithi042 ceph-mon[23723]: pgmap v246: 97 pgs: 97 active+clean; 587 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.7 KiB/s wr, 6 op/s 2024-09-13T12:55:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:57 smithi046 ceph-mon[32268]: pgmap v246: 97 pgs: 97 active+clean; 587 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 2.7 KiB/s wr, 6 op/s 2024-09-13T12:55:57.563 INFO:teuthology.orchestra.run.smithi042.stdout: 5 582 49.63 MB/sec warmup 2 sec latency 30.146 ms 2024-09-13T12:55:58.563 INFO:teuthology.orchestra.run.smithi042.stdout: 5 983 36.50 MB/sec warmup 3 sec latency 23.615 ms 2024-09-13T12:55:59.020 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:55:59 smithi042 ceph-mon[23723]: pgmap v247: 97 pgs: 97 active+clean; 29 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.4 MiB/s wr, 49 op/s 2024-09-13T12:55:59.279 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T12:55:59.366 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:55:59 smithi046 ceph-mon[32268]: pgmap v247: 97 pgs: 97 active+clean; 29 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2.4 MiB/s wr, 49 op/s 2024-09-13T12:55:59.567 INFO:teuthology.orchestra.run.smithi042.stdout: 5 1789 32.86 MB/sec warmup 4 sec latency 39.426 ms 2024-09-13T12:56:00.567 INFO:teuthology.orchestra.run.smithi042.stdout: 5 2667 30.18 MB/sec warmup 5 sec latency 24.123 ms 2024-09-13T12:56:01.037 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:01 smithi042 ceph-mon[23723]: pgmap v248: 97 pgs: 97 active+clean; 74 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.4 MiB/s wr, 243 op/s 2024-09-13T12:56:01.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:01 smithi046 ceph-mon[32268]: pgmap v248: 97 pgs: 97 active+clean; 74 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.4 MiB/s wr, 243 op/s 2024-09-13T12:56:01.567 INFO:teuthology.orchestra.run.smithi042.stdout: 5 3394 30.00 MB/sec warmup 6 sec latency 65.045 ms 2024-09-13T12:56:02.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 3808 28.37 MB/sec warmup 7 sec latency 61.398 ms 2024-09-13T12:56:03.075 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:03 smithi042 ceph-mon[23723]: pgmap v249: 97 pgs: 97 active+clean; 93 MiB data, 130 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.2 MiB/s wr, 403 op/s 2024-09-13T12:56:03.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:03 smithi046 ceph-mon[32268]: pgmap v249: 97 pgs: 97 active+clean; 93 MiB data, 130 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 8.2 MiB/s wr, 403 op/s 2024-09-13T12:56:03.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 4252 25.15 MB/sec warmup 8 sec latency 44.005 ms 2024-09-13T12:56:04.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 4809 23.73 MB/sec warmup 9 sec latency 53.953 ms 2024-09-13T12:56:04.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T12:56:05.090 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:05 smithi042 ceph-mon[23723]: pgmap v250: 97 pgs: 97 active+clean; 124 MiB data, 231 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 12 MiB/s wr, 688 op/s 2024-09-13T12:56:05.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:05 smithi046 ceph-mon[32268]: pgmap v250: 97 pgs: 97 active+clean; 124 MiB data, 231 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 12 MiB/s wr, 688 op/s 2024-09-13T12:56:05.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 5546 22.85 MB/sec warmup 10 sec latency 32.334 ms 2024-09-13T12:56:06.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 5972 21.47 MB/sec warmup 11 sec latency 70.317 ms 2024-09-13T12:56:07.134 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:07 smithi042 ceph-mon[23723]: pgmap v251: 97 pgs: 97 active+clean; 131 MiB data, 231 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 14 MiB/s wr, 812 op/s 2024-09-13T12:56:07.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:07 smithi046 ceph-mon[32268]: pgmap v251: 97 pgs: 97 active+clean; 131 MiB data, 231 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 14 MiB/s wr, 812 op/s 2024-09-13T12:56:07.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 6317 20.92 MB/sec warmup 12 sec latency 85.533 ms 2024-09-13T12:56:08.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 6853 20.94 MB/sec warmup 13 sec latency 42.883 ms 2024-09-13T12:56:09.176 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph mgr module enable nfs --force' 2024-09-13T12:56:09.195 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:09 smithi042 ceph-mon[23723]: pgmap v252: 97 pgs: 97 active+clean; 129 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 14 MiB/s wr, 867 op/s 2024-09-13T12:56:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:09 smithi046 ceph-mon[32268]: pgmap v252: 97 pgs: 97 active+clean; 129 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s rd, 14 MiB/s wr, 867 op/s 2024-09-13T12:56:09.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 7233 20.70 MB/sec warmup 14 sec latency 79.373 ms 2024-09-13T12:56:10.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 7383 19.64 MB/sec warmup 15 sec latency 93.398 ms 2024-09-13T12:56:11.248 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:11 smithi042 ceph-mon[23723]: pgmap v253: 97 pgs: 97 active+clean; 132 MiB data, 284 MiB used, 715 GiB / 715 GiB avail; 7.4 KiB/s rd, 13 MiB/s wr, 965 op/s 2024-09-13T12:56:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:11 smithi046 ceph-mon[32268]: pgmap v253: 97 pgs: 97 active+clean; 132 MiB data, 284 MiB used, 715 GiB / 715 GiB avail; 7.4 KiB/s rd, 13 MiB/s wr, 965 op/s 2024-09-13T12:56:11.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 7714 18.54 MB/sec warmup 16 sec latency 68.816 ms 2024-09-13T12:56:12.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 8008 17.66 MB/sec warmup 17 sec latency 70.085 ms 2024-09-13T12:56:13.042 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:13 smithi042 ceph-mon[23723]: pgmap v254: 97 pgs: 97 active+clean; 133 MiB data, 295 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 9.6 MiB/s wr, 782 op/s 2024-09-13T12:56:13.042 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:13 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/4124628108' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T12:56:13.061 INFO:teuthology.orchestra.run.smithi042.stderr:module 'nfs' is already enabled 2024-09-13T12:56:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:13 smithi046 ceph-mon[32268]: pgmap v254: 97 pgs: 97 active+clean; 133 MiB data, 295 MiB used, 715 GiB / 715 GiB avail; 7.2 KiB/s rd, 9.6 MiB/s wr, 782 op/s 2024-09-13T12:56:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:13 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/4124628108' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "nfs", "force": true}]: dispatch 2024-09-13T12:56:13.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 8842 17.91 MB/sec warmup 18 sec latency 54.324 ms 2024-09-13T12:56:14.046 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:14 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/4124628108' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T12:56:14.046 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:14 smithi042 ceph-mon[23723]: mgrmap e21: smithi042.jhsfyp(active, since 6m), standbys: smithi046.hcgddm 2024-09-13T12:56:14.312 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T12:56:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:14 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/4124628108' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "nfs", "force": true}]': finished 2024-09-13T12:56:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:14 smithi046 ceph-mon[32268]: mgrmap e21: smithi042.jhsfyp(active, since 6m), standbys: smithi046.hcgddm 2024-09-13T12:56:14.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 9685 17.84 MB/sec warmup 19 sec latency 12.730 ms 2024-09-13T12:56:15.212 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:15 smithi042 ceph-mon[23723]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 360 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.7 MiB/s wr, 769 op/s 2024-09-13T12:56:15.366 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:15 smithi046 ceph-mon[32268]: pgmap v255: 97 pgs: 97 active+clean; 142 MiB data, 360 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 9.7 MiB/s wr, 769 op/s 2024-09-13T12:56:15.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 10459 18.49 MB/sec warmup 20 sec latency 32.551 ms 2024-09-13T12:56:16.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 10821 18.39 MB/sec warmup 21 sec latency 43.659 ms 2024-09-13T12:56:17.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:17 smithi042 ceph-mon[23723]: pgmap v256: 97 pgs: 97 active+clean; 141 MiB data, 391 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.3 MiB/s wr, 523 op/s 2024-09-13T12:56:17.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:17 smithi046 ceph-mon[32268]: pgmap v256: 97 pgs: 97 active+clean; 141 MiB data, 391 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.3 MiB/s wr, 523 op/s 2024-09-13T12:56:17.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 11140 17.80 MB/sec warmup 22 sec latency 44.065 ms 2024-09-13T12:56:18.185 INFO:teuthology.orchestra.run.smithi042.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b 2024-09-13T12:56:18.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 11879 17.58 MB/sec warmup 23 sec latency 38.038 ms 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: pgmap v257: 97 pgs: 97 active+clean; 144 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.8 MiB/s wr, 605 op/s 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='client.24375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:56:19.255 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:19.256 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:19 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:19.419 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T12:56:19.419 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T12:56:19.420 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-09-13T12:56:19.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 12369 17.42 MB/sec warmup 24 sec latency 66.741 ms 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: pgmap v257: 97 pgs: 97 active+clean; 144 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 6.8 MiB/s wr, 605 op/s 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='client.24375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:19 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:20.261 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:20 smithi042 ceph-mon[23723]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b 2024-09-13T12:56:20.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 13082 17.17 MB/sec warmup 25 sec latency 24.644 ms 2024-09-13T12:56:20.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:20 smithi046 ceph-mon[32268]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b 2024-09-13T12:56:21.269 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:21 smithi042 ceph-mon[23723]: pgmap v258: 97 pgs: 97 active+clean; 153 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 676 op/s 2024-09-13T12:56:21.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 13723 17.61 MB/sec warmup 26 sec latency 17.814 ms 2024-09-13T12:56:21.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:21 smithi046 ceph-mon[32268]: pgmap v258: 97 pgs: 97 active+clean; 153 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 7.4 MiB/s wr, 676 op/s 2024-09-13T12:56:22.573 INFO:teuthology.orchestra.run.smithi042.stdout: 5 14313 17.71 MB/sec warmup 27 sec latency 50.062 ms 2024-09-13T12:56:23.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:23 smithi042 ceph-mon[23723]: pgmap v259: 97 pgs: 97 active+clean; 152 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 554 op/s 2024-09-13T12:56:23.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 14761 17.42 MB/sec warmup 28 sec latency 23.443 ms 2024-09-13T12:56:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:23 smithi046 ceph-mon[32268]: pgmap v259: 97 pgs: 97 active+clean; 152 MiB data, 451 MiB used, 715 GiB / 715 GiB avail; 7.4 KiB/s rd, 6.3 MiB/s wr, 554 op/s 2024-09-13T12:56:24.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 15402 17.22 MB/sec warmup 29 sec latency 22.441 ms 2024-09-13T12:56:25.296 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:25 smithi042 ceph-mon[23723]: pgmap v260: 97 pgs: 97 active+clean; 162 MiB data, 491 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 876 op/s 2024-09-13T12:56:25.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 16239 17.21 MB/sec warmup 30 sec latency 29.708 ms 2024-09-13T12:56:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:25 smithi046 ceph-mon[32268]: pgmap v260: 97 pgs: 97 active+clean; 162 MiB data, 491 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 9.0 MiB/s wr, 876 op/s 2024-09-13T12:56:26.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 17032 17.45 MB/sec warmup 31 sec latency 17.313 ms 2024-09-13T12:56:27.185 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:56:27.308 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:27 smithi042 ceph-mon[23723]: pgmap v261: 97 pgs: 97 active+clean; 164 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.1 MiB/s wr, 769 op/s 2024-09-13T12:56:27.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 17775 17.91 MB/sec warmup 32 sec latency 16.580 ms 2024-09-13T12:56:27.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:27 smithi046 ceph-mon[32268]: pgmap v261: 97 pgs: 97 active+clean; 164 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.1 MiB/s wr, 769 op/s 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (5m) 35s ago 10m 22.2M - ba2b418f427c 9e454cd0ade6 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (10m) 35s ago 10m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (6m) 38s ago 6m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (5m) 35s ago 7m 51.3M - 8.3.5 dad864ee21e9 ca253f1b9c19 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (69s) 35s ago 68s 16.0M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (66s) 38s ago 65s 13.4M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (11m) 35s ago 11m 459M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:56:28.219 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (6m) 38s ago 6m 410M - 17.2.0 e1d6a67b021e 278d4d510a6d 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (11m) 35s ago 11m 47.3M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (6m) 38s ago 6m 37.0M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (46s) 35s ago 46s 46.9M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (41s) 38s ago 40s 50.7M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (7m) 35s ago 7m 15.3M - 1dbe0e931976 5a333e64c22b 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (6m) 38s ago 6m 18.6M - 1dbe0e931976 7086f6b8e52b 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (5m) 35s ago 5m 43.2M 3217M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (5m) 35s ago 5m 42.4M 3217M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (4m) 35s ago 4m 41.2M 3217M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (4m) 35s ago 4m 39.7M 3217M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (3m) 38s ago 3m 40.1M 3985M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:56:28.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (3m) 38s ago 3m 42.8M 3985M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:56:28.221 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (3m) 38s ago 3m 38.8M 3985M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:56:28.221 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (2m) 38s ago 2m 40.2M 3985M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:56:28.221 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (6m) 35s ago 6m 60.9M - 514e6a882f6e 147d67f2f5fb 2024-09-13T12:56:28.328 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:28 smithi042 ceph-mon[23723]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:28.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 18149 17.64 MB/sec warmup 33 sec latency 60.824 ms 2024-09-13T12:56:28.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:28 smithi046 ceph-mon[32268]: from='client.14726 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:28.789 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:56:28.790 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T12:56:28.791 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:56:28.791 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:56:29.261 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:56:29.261 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b", 2024-09-13T12:56:29.261 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:56:29.261 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T12:56:29.261 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "", 2024-09-13T12:56:29.262 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:b0201b8c79733293453e7f10a10c7fa43119222b image" 2024-09-13T12:56:29.262 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:56:29.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:29 smithi042 ceph-mon[23723]: pgmap v262: 97 pgs: 97 active+clean; 171 MiB data, 595 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 9.7 MiB/s wr, 975 op/s 2024-09-13T12:56:29.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:29 smithi042 ceph-mon[23723]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:29.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:29 smithi042 ceph-mon[23723]: from='client.24387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:29.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:29 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1721870648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:56:29.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 18542 17.22 MB/sec warmup 34 sec latency 37.762 ms 2024-09-13T12:56:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:29 smithi046 ceph-mon[32268]: pgmap v262: 97 pgs: 97 active+clean; 171 MiB data, 595 MiB used, 715 GiB / 715 GiB avail; 9.6 KiB/s rd, 9.7 MiB/s wr, 975 op/s 2024-09-13T12:56:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:29 smithi046 ceph-mon[32268]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:29 smithi046 ceph-mon[32268]: from='client.24387 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:29 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1721870648' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:56:29.786 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:56:30.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:30 smithi042 ceph-mon[23723]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:30.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:30 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/208602956' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:56:30.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 18873 16.93 MB/sec warmup 35 sec latency 43.574 ms 2024-09-13T12:56:30.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:30 smithi046 ceph-mon[32268]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:56:30.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:30 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/208602956' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:56:31.387 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:31 smithi042 ceph-mon[23723]: pgmap v263: 97 pgs: 97 active+clean; 172 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 822 op/s 2024-09-13T12:56:31.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 19263 16.68 MB/sec warmup 36 sec latency 56.409 ms 2024-09-13T12:56:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:31 smithi046 ceph-mon[32268]: pgmap v263: 97 pgs: 97 active+clean; 172 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.0 MiB/s wr, 822 op/s 2024-09-13T12:56:32.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 19975 16.66 MB/sec warmup 37 sec latency 44.275 ms 2024-09-13T12:56:33.405 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:33 smithi042 ceph-mon[23723]: pgmap v264: 97 pgs: 97 active+clean; 177 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 787 op/s 2024-09-13T12:56:33.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 20604 16.72 MB/sec warmup 38 sec latency 31.575 ms 2024-09-13T12:56:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:33 smithi046 ceph-mon[32268]: pgmap v264: 97 pgs: 97 active+clean; 177 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 8.1 MiB/s wr, 787 op/s 2024-09-13T12:56:34.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 21192 16.88 MB/sec warmup 39 sec latency 51.069 ms 2024-09-13T12:56:35.300 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:35 smithi042 ceph-mon[23723]: pgmap v265: 97 pgs: 97 active+clean; 175 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 908 op/s 2024-09-13T12:56:35.574 INFO:teuthology.orchestra.run.smithi042.stdout: 5 21644 16.87 MB/sec warmup 40 sec latency 81.485 ms 2024-09-13T12:56:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:35 smithi046 ceph-mon[32268]: pgmap v265: 97 pgs: 97 active+clean; 175 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 9.5 MiB/s wr, 908 op/s 2024-09-13T12:56:36.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]: dispatch 2024-09-13T12:56:36.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-13T12:56:36.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T12:56:36.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T12:56:36.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:56:36.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:56:36.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 21870 16.59 MB/sec warmup 41 sec latency 62.055 ms 2024-09-13T12:56:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]: dispatch 2024-09-13T12:56:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-09-13T12:56:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-09-13T12:56:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T12:56:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:56:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:56:37.321 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:37 smithi042 ceph-mon[23723]: pgmap v266: 97 pgs: 97 active+clean; 174 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 612 op/s 2024-09-13T12:56:37.321 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:37 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-09-13T12:56:37.321 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:37 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-13T12:56:37.321 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:37 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T12:56:37.321 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:37 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T12:56:37.321 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:37 smithi042 ceph-mon[23723]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T12:56:37.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 22256 16.31 MB/sec warmup 42 sec latency 72.857 ms 2024-09-13T12:56:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:37 smithi046 ceph-mon[32268]: pgmap v266: 97 pgs: 97 active+clean; 174 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 7.4 MiB/s wr, 612 op/s 2024-09-13T12:56:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:37 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.7", "id": [7, 4]}]': finished 2024-09-13T12:56:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:37 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-09-13T12:56:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:37 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-09-13T12:56:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:37 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T12:56:37.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:37 smithi046 ceph-mon[32268]: osdmap e69: 8 total, 8 up, 8 in 2024-09-13T12:56:38.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:38 smithi042 ceph-mon[23723]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T12:56:38.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 22571 16.03 MB/sec warmup 43 sec latency 52.691 ms 2024-09-13T12:56:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:38 smithi046 ceph-mon[32268]: osdmap e70: 8 total, 8 up, 8 in 2024-09-13T12:56:39.390 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:39 smithi042 ceph-mon[23723]: pgmap v269: 97 pgs: 3 peering, 94 active+clean; 186 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.3 MiB/s wr, 660 op/s 2024-09-13T12:56:39.390 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:39 smithi042 ceph-mon[23723]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T12:56:39.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 22846 15.84 MB/sec warmup 44 sec latency 82.609 ms 2024-09-13T12:56:39.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:39 smithi046 ceph-mon[32268]: pgmap v269: 97 pgs: 3 peering, 94 active+clean; 186 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 7.3 MiB/s wr, 660 op/s 2024-09-13T12:56:39.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:39 smithi046 ceph-mon[32268]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2024-09-13T12:56:40.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 23336 15.82 MB/sec warmup 45 sec latency 52.983 ms 2024-09-13T12:56:40.655 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:40 smithi042 ceph-mon[23723]: pgmap v270: 97 pgs: 3 peering, 94 active+clean; 190 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 688 op/s 2024-09-13T12:56:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:40 smithi046 ceph-mon[32268]: pgmap v270: 97 pgs: 3 peering, 94 active+clean; 190 MiB data, 714 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 688 op/s 2024-09-13T12:56:41.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 23862 15.63 MB/sec warmup 46 sec latency 33.909 ms 2024-09-13T12:56:42.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 24274 15.67 MB/sec warmup 47 sec latency 38.603 ms 2024-09-13T12:56:42.693 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:42 smithi042 ceph-mon[23723]: pgmap v271: 97 pgs: 3 peering, 94 active+clean; 188 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.5 MiB/s wr, 494 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:42 smithi046 ceph-mon[32268]: pgmap v271: 97 pgs: 3 peering, 94 active+clean; 188 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 5.5 MiB/s wr, 494 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:43.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 24707 15.69 MB/sec warmup 48 sec latency 41.011 ms 2024-09-13T12:56:43.705 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:43 smithi042 ceph-mon[23723]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T12:56:43.705 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:43 smithi042 ceph-mon[23723]: Cluster is now healthy 2024-09-13T12:56:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:43 smithi046 ceph-mon[32268]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2024-09-13T12:56:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:43 smithi046 ceph-mon[32268]: Cluster is now healthy 2024-09-13T12:56:44.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 25192 15.74 MB/sec warmup 49 sec latency 32.871 ms 2024-09-13T12:56:44.717 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:44 smithi042 ceph-mon[23723]: pgmap v272: 97 pgs: 97 active+clean; 190 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 749 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:44 smithi046 ceph-mon[32268]: pgmap v272: 97 pgs: 97 active+clean; 190 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.0 MiB/s wr, 749 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:45.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 25543 15.58 MB/sec warmup 50 sec latency 42.989 ms 2024-09-13T12:56:46.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 25998 15.35 MB/sec warmup 51 sec latency 58.180 ms 2024-09-13T12:56:46.729 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:46 smithi042 ceph-mon[23723]: pgmap v273: 97 pgs: 97 active+clean; 194 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.0 MiB/s wr, 703 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:46 smithi046 ceph-mon[32268]: pgmap v273: 97 pgs: 97 active+clean; 194 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 7.0 MiB/s wr, 703 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:47.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 26530 15.37 MB/sec warmup 52 sec latency 40.010 ms 2024-09-13T12:56:48.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 27245 15.31 MB/sec warmup 53 sec latency 36.191 ms 2024-09-13T12:56:48.754 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:48 smithi042 ceph-mon[23723]: pgmap v274: 97 pgs: 97 active+clean; 200 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 591 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:48.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:48 smithi046 ceph-mon[32268]: pgmap v274: 97 pgs: 97 active+clean; 200 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 6.1 MiB/s wr, 591 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:49.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 27857 15.43 MB/sec warmup 54 sec latency 42.887 ms 2024-09-13T12:56:50.521 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:50 smithi042 ceph-mon[23723]: pgmap v275: 97 pgs: 97 active+clean; 199 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.9 MiB/s wr, 546 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:50.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 28548 15.61 MB/sec warmup 55 sec latency 65.342 ms 2024-09-13T12:56:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:50 smithi046 ceph-mon[32268]: pgmap v275: 97 pgs: 97 active+clean; 199 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 5.9 MiB/s wr, 546 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:51.548 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:51 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:51.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 28923 15.54 MB/sec warmup 56 sec latency 60.261 ms 2024-09-13T12:56:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:51 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:56:52.568 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:52 smithi042 ceph-mon[23723]: pgmap v276: 97 pgs: 97 active+clean; 206 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 610 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:52.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 29177 15.36 MB/sec warmup 57 sec latency 85.631 ms 2024-09-13T12:56:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:52 smithi046 ceph-mon[32268]: pgmap v276: 97 pgs: 97 active+clean; 206 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.8 MiB/s wr, 610 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:53.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 29521 15.15 MB/sec warmup 58 sec latency 62.459 ms 2024-09-13T12:56:54.575 INFO:teuthology.orchestra.run.smithi042.stdout: 5 29940 15.09 MB/sec warmup 59 sec latency 53.563 ms 2024-09-13T12:56:54.594 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:54 smithi042 ceph-mon[23723]: pgmap v277: 97 pgs: 97 active+clean; 213 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 763 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:54 smithi046 ceph-mon[32268]: pgmap v277: 97 pgs: 97 active+clean; 213 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 763 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T12:56:55.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 30376 14.99 MB/sec warmup 60 sec latency 43.869 ms 2024-09-13T12:56:56.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 30779 14.90 MB/sec warmup 61 sec latency 51.941 ms 2024-09-13T12:56:56.647 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:56 smithi042 ceph-mon[23723]: pgmap v278: 97 pgs: 97 active+clean; 216 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.2 MiB/s wr, 673 op/s 2024-09-13T12:56:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:56 smithi046 ceph-mon[32268]: pgmap v278: 97 pgs: 97 active+clean; 216 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.2 MiB/s wr, 673 op/s 2024-09-13T12:56:57.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 31381 14.98 MB/sec warmup 62 sec latency 31.638 ms 2024-09-13T12:56:58.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 31796 15.00 MB/sec warmup 63 sec latency 63.801 ms 2024-09-13T12:56:58.670 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:56:58 smithi042 ceph-mon[23723]: pgmap v279: 97 pgs: 97 active+clean; 211 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.1 MiB/s wr, 673 op/s 2024-09-13T12:56:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:56:58 smithi046 ceph-mon[32268]: pgmap v279: 97 pgs: 97 active+clean; 211 MiB data, 912 MiB used, 714 GiB / 715 GiB avail; 8.4 KiB/s rd, 7.1 MiB/s wr, 673 op/s 2024-09-13T12:56:59.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 32163 14.96 MB/sec warmup 64 sec latency 70.950 ms 2024-09-13T12:57:00.187 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:57:00.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 32364 14.83 MB/sec warmup 65 sec latency 120.999 ms 2024-09-13T12:57:00.979 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:57:00.979 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (6m) 68s ago 10m 22.2M - ba2b418f427c 9e454cd0ade6 2024-09-13T12:57:00.979 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (10m) 68s ago 10m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (7m) 70s ago 7m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (6m) 68s ago 7m 51.3M - 8.3.5 dad864ee21e9 ca253f1b9c19 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (102s) 68s ago 100s 16.0M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (98s) 70s ago 98s 13.4M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (11m) 68s ago 11m 459M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (7m) 70s ago 7m 410M - 17.2.0 e1d6a67b021e 278d4d510a6d 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (11m) 68s ago 12m 47.3M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (7m) 70s ago 7m 37.0M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (79s) 68s ago 79s 46.9M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (74s) 70s ago 73s 50.7M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:57:00.980 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (7m) 68s ago 7m 15.3M - 1dbe0e931976 5a333e64c22b 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (6m) 70s ago 6m 18.6M - 1dbe0e931976 7086f6b8e52b 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (6m) 68s ago 6m 43.2M 3217M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (5m) 68s ago 5m 42.4M 3217M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (5m) 68s ago 5m 41.2M 3217M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (4m) 68s ago 4m 39.7M 3217M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (4m) 70s ago 4m 40.1M 3985M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (4m) 70s ago 4m 42.8M 3985M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (3m) 70s ago 3m 38.8M 3985M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (3m) 70s ago 3m 40.2M 3985M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:57:00.981 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (6m) 68s ago 6m 60.9M - 514e6a882f6e 147d67f2f5fb 2024-09-13T12:57:01.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: pgmap v280: 97 pgs: 97 active+clean; 215 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 650 op/s 2024-09-13T12:57:01.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:01.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:01.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: Upgrade: Target is version 19.3.0-4917-gb0201b8c (unknown) 2024-09-13T12:57:01.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, digests ['quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b'] 2024-09-13T12:57:01.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:01.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:01.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:01 smithi042 ceph-mon[23723]: Upgrade: Need to upgrade myself (mgr.smithi042.jhsfyp) 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:57:01.432 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:57:01.433 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:57:01.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 32660 14.67 MB/sec warmup 66 sec latency 151.635 ms 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: pgmap v280: 97 pgs: 97 active+clean; 215 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.6 MiB/s wr, 650 op/s 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: from='client.14750 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: Upgrade: Target is version 19.3.0-4917-gb0201b8c (unknown) 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, digests ['quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b'] 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:01.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:01 smithi046 ceph-mon[32268]: Upgrade: Need to upgrade myself (mgr.smithi042.jhsfyp) 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "0/23 daemons upgraded", 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b image on host smithi046" 2024-09-13T12:57:01.819 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:57:02.392 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:02 smithi042 ceph-mon[23723]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:02.392 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:02 smithi042 ceph-mon[23723]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b on smithi046 2024-09-13T12:57:02.392 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:02 smithi042 ceph-mon[23723]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:02.392 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:02 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/161264060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:57:02.415 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:57:02.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 32969 14.54 MB/sec warmup 67 sec latency 80.227 ms 2024-09-13T12:57:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:02 smithi046 ceph-mon[32268]: from='client.14754 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:02 smithi046 ceph-mon[32268]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b on smithi046 2024-09-13T12:57:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:02 smithi046 ceph-mon[32268]: from='client.14758 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:02 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/161264060' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:57:03.396 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:03 smithi042 ceph-mon[23723]: pgmap v281: 97 pgs: 97 active+clean; 217 MiB data, 987 MiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T12:57:03.396 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:03 smithi042 ceph-mon[23723]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:03.396 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:03 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1831466241' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:57:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 33527 14.49 MB/sec warmup 68 sec latency 42.636 ms 2024-09-13T12:57:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:03 smithi046 ceph-mon[32268]: pgmap v281: 97 pgs: 97 active+clean; 217 MiB data, 987 MiB used, 714 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 621 op/s 2024-09-13T12:57:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:03 smithi046 ceph-mon[32268]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:03 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1831466241' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:57:04.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 34073 14.45 MB/sec warmup 69 sec latency 50.355 ms 2024-09-13T12:57:05.424 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:05 smithi042 ceph-mon[23723]: pgmap v282: 97 pgs: 97 active+clean; 229 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-09-13T12:57:05.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 34499 14.43 MB/sec warmup 70 sec latency 75.040 ms 2024-09-13T12:57:05.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:05 smithi046 ceph-mon[32268]: pgmap v282: 97 pgs: 97 active+clean; 229 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 7.2 MiB/s wr, 653 op/s 2024-09-13T12:57:06.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 34898 14.41 MB/sec warmup 71 sec latency 47.281 ms 2024-09-13T12:57:07.476 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:07 smithi042 ceph-mon[23723]: pgmap v283: 97 pgs: 97 active+clean; 230 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 623 op/s 2024-09-13T12:57:07.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 35309 14.42 MB/sec warmup 72 sec latency 50.429 ms 2024-09-13T12:57:07.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:07 smithi046 ceph-mon[32268]: pgmap v283: 97 pgs: 97 active+clean; 230 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.8 MiB/s wr, 623 op/s 2024-09-13T12:57:08.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 35825 14.47 MB/sec warmup 73 sec latency 54.290 ms 2024-09-13T12:57:09.501 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:09 smithi042 ceph-mon[23723]: pgmap v284: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-13T12:57:09.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 36386 14.44 MB/sec warmup 74 sec latency 43.726 ms 2024-09-13T12:57:09.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:09 smithi046 ceph-mon[32268]: pgmap v284: 97 pgs: 97 active+clean; 228 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-13T12:57:10.532 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:10 smithi042 ceph-mon[23723]: pgmap v285: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 550 op/s 2024-09-13T12:57:10.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 36728 14.32 MB/sec warmup 75 sec latency 53.110 ms 2024-09-13T12:57:10.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:10 smithi046 ceph-mon[32268]: pgmap v285: 97 pgs: 97 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.1 MiB/s wr, 550 op/s 2024-09-13T12:57:11.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 37160 14.24 MB/sec warmup 76 sec latency 63.102 ms 2024-09-13T12:57:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:12 smithi042 ceph-mon[23723]: pgmap v286: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-09-13T12:57:12.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 37726 14.24 MB/sec warmup 77 sec latency 46.808 ms 2024-09-13T12:57:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:12 smithi046 ceph-mon[32268]: pgmap v286: 97 pgs: 97 active+clean; 239 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 557 op/s 2024-09-13T12:57:13.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 38466 14.36 MB/sec warmup 78 sec latency 43.873 ms 2024-09-13T12:57:14.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 39035 14.47 MB/sec warmup 79 sec latency 43.308 ms 2024-09-13T12:57:14.589 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:14 smithi042 ceph-mon[23723]: pgmap v287: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 691 op/s 2024-09-13T12:57:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:14 smithi046 ceph-mon[32268]: pgmap v287: 97 pgs: 97 active+clean; 237 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.2 MiB/s wr, 691 op/s 2024-09-13T12:57:15.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 39385 14.40 MB/sec warmup 80 sec latency 64.892 ms 2024-09-13T12:57:16.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 39753 14.31 MB/sec warmup 81 sec latency 50.226 ms 2024-09-13T12:57:16.614 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:16 smithi042 ceph-mon[23723]: pgmap v288: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 616 op/s 2024-09-13T12:57:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:16 smithi046 ceph-mon[32268]: pgmap v288: 97 pgs: 97 active+clean; 241 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.2 MiB/s wr, 616 op/s 2024-09-13T12:57:17.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 40075 14.21 MB/sec warmup 82 sec latency 86.671 ms 2024-09-13T12:57:18.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 40461 14.13 MB/sec warmup 83 sec latency 72.681 ms 2024-09-13T12:57:18.660 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:18 smithi042 ceph-mon[23723]: pgmap v289: 97 pgs: 97 active+clean; 245 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.6 MiB/s wr, 533 op/s 2024-09-13T12:57:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:18 smithi046 ceph-mon[32268]: pgmap v289: 97 pgs: 97 active+clean; 245 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 6.6 MiB/s wr, 533 op/s 2024-09-13T12:57:19.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 40882 14.09 MB/sec warmup 84 sec latency 34.162 ms 2024-09-13T12:57:20.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 41206 14.02 MB/sec warmup 85 sec latency 53.478 ms 2024-09-13T12:57:20.722 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:20 smithi042 ceph-mon[23723]: pgmap v290: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 605 op/s 2024-09-13T12:57:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:20 smithi046 ceph-mon[32268]: pgmap v290: 97 pgs: 97 active+clean; 248 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 6.9 MiB/s wr, 605 op/s 2024-09-13T12:57:21.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 41525 13.95 MB/sec warmup 86 sec latency 47.490 ms 2024-09-13T12:57:22.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 41714 13.84 MB/sec warmup 87 sec latency 83.284 ms 2024-09-13T12:57:22.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:22 smithi042 ceph-mon[23723]: pgmap v291: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 708 op/s 2024-09-13T12:57:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:22 smithi046 ceph-mon[32268]: pgmap v291: 97 pgs: 97 active+clean; 253 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 6.6 MiB/s wr, 708 op/s 2024-09-13T12:57:23.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 42040 13.81 MB/sec warmup 88 sec latency 74.525 ms 2024-09-13T12:57:24.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 42350 13.80 MB/sec warmup 89 sec latency 46.685 ms 2024-09-13T12:57:24.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:24 smithi042 ceph-mon[23723]: pgmap v292: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 699 op/s 2024-09-13T12:57:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:24 smithi046 ceph-mon[32268]: pgmap v292: 97 pgs: 97 active+clean; 252 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 699 op/s 2024-09-13T12:57:25.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 42703 13.81 MB/sec warmup 90 sec latency 65.081 ms 2024-09-13T12:57:26.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 43028 13.75 MB/sec warmup 91 sec latency 40.619 ms 2024-09-13T12:57:26.755 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:26 smithi042 ceph-mon[23723]: pgmap v293: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-09-13T12:57:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:26 smithi046 ceph-mon[32268]: pgmap v293: 97 pgs: 97 active+clean; 256 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 6.0 MiB/s wr, 577 op/s 2024-09-13T12:57:27.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 43300 13.64 MB/sec warmup 92 sec latency 42.167 ms 2024-09-13T12:57:28.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 43624 13.56 MB/sec warmup 93 sec latency 82.961 ms 2024-09-13T12:57:28.754 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:28 smithi042 ceph-mon[23723]: pgmap v294: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 512 op/s 2024-09-13T12:57:28.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:28 smithi046 ceph-mon[32268]: pgmap v294: 97 pgs: 97 active+clean; 261 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 512 op/s 2024-09-13T12:57:29.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 43958 13.48 MB/sec warmup 94 sec latency 85.911 ms 2024-09-13T12:57:30.544 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:30 smithi042 ceph-mon[23723]: pgmap v295: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.3 MiB/s wr, 559 op/s 2024-09-13T12:57:30.576 INFO:teuthology.orchestra.run.smithi042.stdout: 5 44343 13.45 MB/sec warmup 95 sec latency 73.883 ms 2024-09-13T12:57:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:30 smithi046 ceph-mon[32268]: pgmap v295: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 5.3 MiB/s wr, 559 op/s 2024-09-13T12:57:31.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 44850 13.43 MB/sec warmup 96 sec latency 32.974 ms 2024-09-13T12:57:32.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 45258 13.39 MB/sec warmup 97 sec latency 45.705 ms 2024-09-13T12:57:32.922 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:57:33.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: pgmap v296: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 453 op/s 2024-09-13T12:57:33.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: Upgrade: Updating mgr.smithi046.hcgddm 2024-09-13T12:57:33.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:33.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:57:33.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:57:33.378 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:57:33.379 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:33 smithi042 ceph-mon[23723]: Deploying daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T12:57:33.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 45643 13.42 MB/sec warmup 98 sec latency 43.874 ms 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: pgmap v296: 97 pgs: 97 active+clean; 261 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 453 op/s 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: Upgrade: Updating mgr.smithi046.hcgddm 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:57:33.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:33 smithi046 ceph-mon[32268]: Deploying daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T12:57:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:57:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (6m) 101s ago 11m 22.2M - ba2b418f427c 9e454cd0ade6 2024-09-13T12:57:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (11m) 101s ago 11m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:57:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (7m) 103s ago 7m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:57:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (6m) 101s ago 8m 51.3M - 8.3.5 dad864ee21e9 ca253f1b9c19 2024-09-13T12:57:33.857 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (2m) 101s ago 2m 16.0M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (2m) 103s ago 2m 13.4M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (12m) 101s ago 12m 459M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (7m) 103s ago 7m 410M - 17.2.0 e1d6a67b021e 278d4d510a6d 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (12m) 101s ago 12m 47.3M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (7m) 103s ago 7m 37.0M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (112s) 101s ago 111s 46.9M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (106s) 103s ago 106s 50.7M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (8m) 101s ago 8m 15.3M - 1dbe0e931976 5a333e64c22b 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (7m) 103s ago 7m 18.6M - 1dbe0e931976 7086f6b8e52b 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (6m) 101s ago 6m 43.2M 3217M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (6m) 101s ago 6m 42.4M 3217M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:57:33.858 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (5m) 101s ago 5m 41.2M 3217M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:57:33.859 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (5m) 101s ago 5m 39.7M 3217M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:57:33.859 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (4m) 103s ago 4m 40.1M 3985M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:57:33.859 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (4m) 103s ago 4m 42.8M 3985M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:57:33.859 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (4m) 103s ago 4m 38.8M 3985M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:57:33.859 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (3m) 103s ago 3m 40.2M 3985M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:57:33.859 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (7m) 101s ago 7m 60.9M - 514e6a882f6e 147d67f2f5fb 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:57:34.304 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:57:34.305 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:57:34.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:34 smithi042 ceph-mon[23723]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:34 smithi046 ceph-mon[32268]: from='client.14774 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:34.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 46036 13.43 MB/sec warmup 99 sec latency 57.724 ms 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "0/23 daemons upgraded", 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mgr daemons" 2024-09-13T12:57:34.702 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:57:35.201 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:35 smithi042 ceph-mon[23723]: from='client.24419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:35.201 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:35 smithi042 ceph-mon[23723]: pgmap v297: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 457 op/s 2024-09-13T12:57:35.201 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:35 smithi042 ceph-mon[23723]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:35.201 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:35 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2169648567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:57:35.349 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:57:35.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 46571 13.46 MB/sec warmup 100 sec latency 49.159 ms 2024-09-13T12:57:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:35 smithi046 ceph-mon[32268]: from='client.24419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:35 smithi046 ceph-mon[32268]: pgmap v297: 97 pgs: 97 active+clean; 262 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 5.1 MiB/s wr, 457 op/s 2024-09-13T12:57:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:35 smithi046 ceph-mon[32268]: from='client.14782 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:35 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2169648567' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:57:36.214 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:36 smithi042 ceph-mon[23723]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:36.214 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:36 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/403212000' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:57:36.214 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:57:36.214 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:36 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:57:36.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 47167 13.43 MB/sec warmup 101 sec latency 77.413 ms 2024-09-13T12:57:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:36 smithi046 ceph-mon[32268]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:57:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:36 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/403212000' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:57:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:57:36.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:36 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:57:37.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:37 smithi042 ceph-mon[23723]: pgmap v298: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 474 op/s 2024-09-13T12:57:37.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:37 smithi046 ceph-mon[32268]: pgmap v298: 97 pgs: 97 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 474 op/s 2024-09-13T12:57:37.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 47737 13.42 MB/sec warmup 102 sec latency 114.366 ms 2024-09-13T12:57:38.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 48550 13.47 MB/sec warmup 103 sec latency 25.264 ms 2024-09-13T12:57:39.286 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:39 smithi042 ceph-mon[23723]: pgmap v299: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.1 MiB/s wr, 622 op/s 2024-09-13T12:57:39.286 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:39.286 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:39.286 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:57:39.286 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:39 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:57:39.456 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:39 smithi046 ceph-mon[32268]: pgmap v299: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 6.1 MiB/s wr, 622 op/s 2024-09-13T12:57:39.456 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:39.456 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:39.456 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:57:39.456 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:39 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:57:39.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 49294 13.61 MB/sec warmup 104 sec latency 20.673 ms 2024-09-13T12:57:40.549 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:40 smithi042 ceph-mon[23723]: pgmap v300: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 612 op/s 2024-09-13T12:57:40.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 49953 13.68 MB/sec warmup 105 sec latency 46.277 ms 2024-09-13T12:57:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:40 smithi046 ceph-mon[32268]: pgmap v300: 97 pgs: 97 active+clean; 278 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 612 op/s 2024-09-13T12:57:41.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 50315 13.64 MB/sec warmup 106 sec latency 86.411 ms 2024-09-13T12:57:42.570 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:42 smithi042 ceph-mon[23723]: pgmap v301: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-09-13T12:57:42.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 50818 13.57 MB/sec warmup 107 sec latency 42.960 ms 2024-09-13T12:57:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:42 smithi046 ceph-mon[32268]: pgmap v301: 97 pgs: 97 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.4 KiB/s rd, 5.7 MiB/s wr, 535 op/s 2024-09-13T12:57:43.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 51340 13.55 MB/sec warmup 108 sec latency 75.160 ms 2024-09-13T12:57:43.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:43 smithi042 ceph-mon[23723]: Standby manager daemon smithi046.hcgddm restarted 2024-09-13T12:57:43.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:43 smithi042 ceph-mon[23723]: Standby manager daemon smithi046.hcgddm started 2024-09-13T12:57:43.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:43 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T12:57:43.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:43 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:57:43.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:43 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T12:57:43.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:43 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:57:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:43 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm restarted 2024-09-13T12:57:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:43 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm started 2024-09-13T12:57:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T12:57:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:57:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T12:57:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:57:44.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 52225 13.66 MB/sec warmup 109 sec latency 31.221 ms 2024-09-13T12:57:44.584 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:44 smithi042 ceph-mon[23723]: pgmap v302: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 810 op/s 2024-09-13T12:57:44.584 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:44 smithi042 ceph-mon[23723]: mgrmap e22: smithi042.jhsfyp(active, since 8m), standbys: smithi046.hcgddm 2024-09-13T12:57:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:44 smithi046 ceph-mon[32268]: pgmap v302: 97 pgs: 97 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 8.4 MiB/s wr, 810 op/s 2024-09-13T12:57:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:44 smithi046 ceph-mon[32268]: mgrmap e22: smithi042.jhsfyp(active, since 8m), standbys: smithi046.hcgddm 2024-09-13T12:57:45.577 INFO:teuthology.orchestra.run.smithi042.stdout: 5 52839 13.73 MB/sec warmup 110 sec latency 25.399 ms 2024-09-13T12:57:46.581 INFO:teuthology.orchestra.run.smithi042.stdout: 5 53411 13.77 MB/sec warmup 111 sec latency 26.143 ms 2024-09-13T12:57:46.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:46 smithi042 ceph-mon[23723]: pgmap v303: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.2 MiB/s wr, 830 op/s 2024-09-13T12:57:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:46 smithi046 ceph-mon[32268]: pgmap v303: 97 pgs: 97 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.2 MiB/s wr, 830 op/s 2024-09-13T12:57:47.585 INFO:teuthology.orchestra.run.smithi042.stdout: 5 53920 13.79 MB/sec warmup 112 sec latency 104.536 ms 2024-09-13T12:57:48.581 INFO:teuthology.orchestra.run.smithi042.stdout: 5 54503 13.73 MB/sec warmup 113 sec latency 114.221 ms 2024-09-13T12:57:48.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:48 smithi042 ceph-mon[23723]: pgmap v304: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.4 MiB/s wr, 768 op/s 2024-09-13T12:57:48.791 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:48 smithi046 ceph-mon[32268]: pgmap v304: 97 pgs: 97 active+clean; 291 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.1 KiB/s rd, 8.4 MiB/s wr, 768 op/s 2024-09-13T12:57:49.581 INFO:teuthology.orchestra.run.smithi042.stdout: 5 55274 13.81 MB/sec warmup 114 sec latency 23.481 ms 2024-09-13T12:57:50.581 INFO:teuthology.orchestra.run.smithi042.stdout: 5 56020 13.84 MB/sec warmup 115 sec latency 35.688 ms 2024-09-13T12:57:50.637 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:50 smithi042 ceph-mon[23723]: pgmap v305: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 696 op/s 2024-09-13T12:57:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:50 smithi046 ceph-mon[32268]: pgmap v305: 97 pgs: 97 active+clean; 297 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 8.2 MiB/s wr, 696 op/s 2024-09-13T12:57:51.581 INFO:teuthology.orchestra.run.smithi042.stdout: 5 56429 13.85 MB/sec warmup 116 sec latency 41.012 ms 2024-09-13T12:57:52.581 INFO:teuthology.orchestra.run.smithi042.stdout: 5 56918 13.85 MB/sec warmup 117 sec latency 67.011 ms 2024-09-13T12:57:52.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:52 smithi046 ceph-mon[32268]: pgmap v306: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-09-13T12:57:52.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:52 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:52.672 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:52 smithi042 ceph-mon[23723]: pgmap v306: 97 pgs: 97 active+clean; 300 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 7.8 MiB/s wr, 736 op/s 2024-09-13T12:57:52.672 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:52 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:53.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 57386 13.88 MB/sec warmup 118 sec latency 31.089 ms 2024-09-13T12:57:54.185 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:53 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:54.238 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:53 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:54.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 57787 13.80 MB/sec warmup 119 sec latency 53.007 ms 2024-09-13T12:57:55.193 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:54 smithi042 ceph-mon[23723]: pgmap v307: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.9 MiB/s wr, 974 op/s 2024-09-13T12:57:55.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:54 smithi046 ceph-mon[32268]: pgmap v307: 97 pgs: 97 active+clean; 307 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 9.9 MiB/s wr, 974 op/s 2024-09-13T12:57:56.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 58822 14.48 MB/sec execute 1 sec latency 41.865 ms 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: pgmap v308: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 803 op/s 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr fail", "who": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:57:57.015 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:56 smithi042 ceph-mon[23723]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: pgmap v308: 97 pgs: 97 active+clean; 313 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 8.3 MiB/s wr, 803 op/s 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr fail", "who": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:57:57.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:56 smithi046 ceph-mon[32268]: osdmap e71: 8 total, 8 up, 8 in 2024-09-13T12:57:57.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 59397 14.86 MB/sec execute 2 sec latency 51.961 ms 2024-09-13T12:57:58.024 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: Upgrade: Need to upgrade myself (mgr.smithi042.jhsfyp) 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: Failing over to other MGR 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "mgr fail", "who": "smithi042.jhsfyp"}]': finished 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: mgrmap e23: smithi046.hcgddm(active, starting, since 0.996381s) 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"}]: dispatch 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"}]: dispatch 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:57:58.025 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:57:58.026 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:57:58.027 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:57:58.027 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:57 smithi042 ceph-mon[23723]: Manager daemon smithi046.hcgddm is now available 2024-09-13T12:57:58.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: Upgrade: Need to upgrade myself (mgr.smithi042.jhsfyp) 2024-09-13T12:57:58.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: Failing over to other MGR 2024-09-13T12:57:58.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14300 172.21.15.42:0/1041975266' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "mgr fail", "who": "smithi042.jhsfyp"}]': finished 2024-09-13T12:57:58.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: mgrmap e23: smithi046.hcgddm(active, starting, since 0.996381s) 2024-09-13T12:57:58.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:57:58.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:57:58.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:57:58.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:57:58.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:57 smithi046 ceph-mon[32268]: Manager daemon smithi046.hcgddm is now available 2024-09-13T12:57:58.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 59873 15.33 MB/sec execute 3 sec latency 38.367 ms 2024-09-13T12:57:59.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 60414 16.16 MB/sec execute 4 sec latency 30.599 ms 2024-09-13T12:57:59.787 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: mgrmap e24: smithi046.hcgddm(active, since 1.99913s) 2024-09-13T12:57:59.787 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: pgmap v3: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T12:57:59.787 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.787 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/trash_purge_schedule"}]: dispatch 2024-09-13T12:57:59.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:57:59 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/trash_purge_schedule"}]: dispatch 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: mgrmap e24: smithi046.hcgddm(active, since 1.99913s) 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: pgmap v3: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/trash_purge_schedule"}]: dispatch 2024-09-13T12:57:59.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:57:59 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi046.hcgddm/trash_purge_schedule"}]: dispatch 2024-09-13T12:58:00.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 60864 15.49 MB/sec execute 5 sec latency 40.489 ms 2024-09-13T12:58:00.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:00 smithi042 ceph-mon[23723]: Checking for cert/key for grafana.smithi042 2024-09-13T12:58:00.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:00 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:58:00.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:00 smithi042 ceph-mon[23723]: pgmap v4: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T12:58:00.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:00 smithi042 ceph-mon[23723]: Deploying cephadm binary to smithi046 2024-09-13T12:58:00.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:00 smithi042 ceph-mon[23723]: Deploying cephadm binary to smithi042 2024-09-13T12:58:00.793 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:00 smithi042 ceph-mon[23723]: mgrmap e25: smithi046.hcgddm(active, since 3s) 2024-09-13T12:58:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:00 smithi046 ceph-mon[32268]: Checking for cert/key for grafana.smithi042 2024-09-13T12:58:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:00 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:58:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:00 smithi046 ceph-mon[32268]: pgmap v4: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T12:58:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:00 smithi046 ceph-mon[32268]: Deploying cephadm binary to smithi046 2024-09-13T12:58:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:00 smithi046 ceph-mon[32268]: Deploying cephadm binary to smithi042 2024-09-13T12:58:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:00 smithi046 ceph-mon[32268]: mgrmap e25: smithi046.hcgddm(active, since 3s) 2024-09-13T12:58:01.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 61313 14.16 MB/sec execute 6 sec latency 28.879 ms 2024-09-13T12:58:02.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 61704 13.19 MB/sec execute 7 sec latency 32.816 ms 2024-09-13T12:58:02.785 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: [13/Sep/2024:12:58:01] ENGINE Bus STARTING 2024-09-13T12:58:02.785 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: [13/Sep/2024:12:58:01] ENGINE Serving on http://172.21.15.46:8765 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: [13/Sep/2024:12:58:01] ENGINE Serving on https://172.21.15.46:7150 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: [13/Sep/2024:12:58:01] ENGINE Bus STARTED 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: [13/Sep/2024:12:58:01] ENGINE Error in HTTPServer.serve 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Traceback (most recent call last): 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._connections.run(self.expiration_interval) 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._run(expiration_interval) 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T12:58:02.786 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: s = self.context.wrap_socket( 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: return self.sslsocket_class._create( 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self.do_handshake() 2024-09-13T12:58:02.787 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T12:58:02.788 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._sslobj.do_handshake() 2024-09-13T12:58:02.788 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T12:58:02.788 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: pgmap v5: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T12:58:02.788 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:02 smithi046 ceph-mon[32268]: mgrmap e26: smithi046.hcgddm(active, since 5s) 2024-09-13T12:58:02.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: [13/Sep/2024:12:58:01] ENGINE Bus STARTING 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: [13/Sep/2024:12:58:01] ENGINE Serving on http://172.21.15.46:8765 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: [13/Sep/2024:12:58:01] ENGINE Serving on https://172.21.15.46:7150 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: [13/Sep/2024:12:58:01] ENGINE Bus STARTED 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: [13/Sep/2024:12:58:01] ENGINE Error in HTTPServer.serve 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Traceback (most recent call last): 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._run(expiration_interval) 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T12:58:02.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self.do_handshake() 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: pgmap v5: 97 pgs: 97 active+clean; 320 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-13T12:58:02.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:02 smithi042 ceph-mon[23723]: mgrmap e26: smithi046.hcgddm(active, since 5s) 2024-09-13T12:58:03.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 62210 13.58 MB/sec execute 8 sec latency 43.257 ms 2024-09-13T12:58:04.585 INFO:teuthology.orchestra.run.smithi042.stdout: 5 62677 12.90 MB/sec execute 9 sec latency 73.363 ms 2024-09-13T12:58:04.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:04 smithi042 ceph-mon[23723]: pgmap v6: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.6 MiB/s wr, 531 op/s 2024-09-13T12:58:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:04 smithi046 ceph-mon[32268]: pgmap v6: 97 pgs: 97 active+clean; 323 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 46 KiB/s rd, 5.6 MiB/s wr, 531 op/s 2024-09-13T12:58:05.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 63068 12.82 MB/sec execute 10 sec latency 78.139 ms 2024-09-13T12:58:05.976 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:58:06.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:06 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:06 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:06 smithi042 ceph-mon[23723]: pgmap v7: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 5.1 MiB/s wr, 501 op/s 2024-09-13T12:58:06.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:06 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:06 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.168 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:06 smithi042 ceph-mon[23723]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:06 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:06 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:06 smithi046 ceph-mon[32268]: pgmap v7: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 5.1 MiB/s wr, 501 op/s 2024-09-13T12:58:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:06 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:06 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:06 smithi046 ceph-mon[32268]: from='client.14824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:06.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 63467 13.05 MB/sec execute 11 sec latency 46.826 ms 2024-09-13T12:58:07.175 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:07 smithi042 ceph-mon[23723]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:07.218 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:58:07.218 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (7m) 2m ago 11m 22.2M - ba2b418f427c 9e454cd0ade6 2024-09-13T12:58:07.218 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (11m) 2m ago 11m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:58:07.218 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (8m) 2s ago 8m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:58:07.218 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (7m) 2m ago 8m 51.3M - 8.3.5 dad864ee21e9 ca253f1b9c19 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (2m) 2m ago 2m 16.0M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (2m) 2s ago 2m 14.2M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (13m) 2m ago 13m 459M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (31s) 2s ago 8m 495M - 19.3.0-4917-gb0201b8c 1641fe80ce72 5409c2ac218e 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (13m) 2m ago 13m 47.3M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (8m) 2s ago 8m 38.6M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (2m) 2m ago 2m 46.9M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (2m) 2s ago 2m 75.7M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (8m) 2m ago 8m 15.3M - 1dbe0e931976 5a333e64c22b 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (8m) 2s ago 8m 18.9M - 1.3.1 1dbe0e931976 7086f6b8e52b 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (7m) 2m ago 7m 43.2M 3217M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (6m) 2m ago 6m 42.4M 3217M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (6m) 2m ago 6m 41.2M 3217M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:58:07.219 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (5m) 2m ago 5m 39.7M 3217M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:58:07.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (5m) 2s ago 5m 244M 3985M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:58:07.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (5m) 2s ago 5m 272M 3985M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:58:07.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (4m) 2s ago 4m 246M 3985M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:58:07.220 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (4m) 2s ago 4m 336M 3985M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:58:07.220 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (7m) 2m ago 7m 60.9M - 514e6a882f6e 147d67f2f5fb 2024-09-13T12:58:07.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:07 smithi046 ceph-mon[32268]: from='client.14828 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:07.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 63845 12.94 MB/sec execute 12 sec latency 43.222 ms 2024-09-13T12:58:07.890 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:58:07.890 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:58:07.890 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:58:07.891 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T12:58:07.892 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T12:58:07.892 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:58:07.892 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/crt"}]: dispatch 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: Standby manager daemon smithi042.jhsfyp started 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/key"}]: dispatch 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:58:08.183 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: pgmap v8: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 390 op/s 2024-09-13T12:58:08.184 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:08 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/858897574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: from='client.14832 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/crt"}]: dispatch 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: Standby manager daemon smithi042.jhsfyp started 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/key"}]: dispatch 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/587111676' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: pgmap v8: 97 pgs: 97 active+clean; 327 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 390 op/s 2024-09-13T12:58:08.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:08 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/858897574' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:58:08.423 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:58:08.423 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T12:58:08.423 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:58:08.423 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T12:58:08.423 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T12:58:08.423 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T12:58:08.424 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T12:58:08.424 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T12:58:08.424 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:58:08.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 64261 12.95 MB/sec execute 13 sec latency 31.071 ms 2024-09-13T12:58:09.033 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:58:09.190 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:09 smithi042 ceph-mon[23723]: mgrmap e27: smithi046.hcgddm(active, since 11s), standbys: smithi042.jhsfyp 2024-09-13T12:58:09.190 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:09 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:58:09.190 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:09 smithi042 ceph-mon[23723]: from='client.24465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:09.190 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:09 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3726758248' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:58:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:09 smithi046 ceph-mon[32268]: mgrmap e27: smithi046.hcgddm(active, since 11s), standbys: smithi042.jhsfyp 2024-09-13T12:58:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:09 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:58:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:09 smithi046 ceph-mon[32268]: from='client.24465 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:09.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:09 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3726758248' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:58:09.583 INFO:teuthology.orchestra.run.smithi042.stdout: 5 64609 12.74 MB/sec execute 14 sec latency 43.246 ms 2024-09-13T12:58:10.472 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.472 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.472 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.472 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.472 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi046 to 2705M 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 MiB/s wr, 603 op/s 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 65032 12.36 MB/sec execute 15 sec latency 76.949 ms 2024-09-13T12:58:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi046 to 2705M 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 MiB/s wr, 603 op/s 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:11.582 INFO:teuthology.orchestra.run.smithi042.stdout: 5 65337 11.88 MB/sec execute 16 sec latency 59.709 ms 2024-09-13T12:58:12.046 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.046 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: Adjusting osd_memory_target on smithi042 to 1937M 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:58:12.047 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:11 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: Adjusting osd_memory_target on smithi042 to 1937M 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:11 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:12.591 INFO:teuthology.orchestra.run.smithi042.stdout: 5 65787 12.02 MB/sec execute 17 sec latency 38.672 ms 2024-09-13T12:58:13.042 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:58:13.042 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:58:13.042 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:58:13.042 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:58:13.043 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:58:13.043 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:58:13.043 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-13T12:58:13.043 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Reconfiguring alertmanager.smithi042 (dependencies changed)... 2024-09-13T12:58:13.043 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:12 smithi042 ceph-mon[23723]: Deploying daemon alertmanager.smithi042 on smithi042 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 5.7 MiB/s wr, 553 op/s 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Reconfiguring alertmanager.smithi042 (dependencies changed)... 2024-09-13T12:58:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:12 smithi046 ceph-mon[32268]: Deploying daemon alertmanager.smithi042 on smithi042 2024-09-13T12:58:13.591 INFO:teuthology.orchestra.run.smithi042.stdout: 5 66295 12.08 MB/sec execute 18 sec latency 46.841 ms 2024-09-13T12:58:14.592 INFO:teuthology.orchestra.run.smithi042.stdout: 5 66800 12.03 MB/sec execute 19 sec latency 43.488 ms 2024-09-13T12:58:15.080 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:14 smithi042 ceph-mon[23723]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.7 MiB/s wr, 844 op/s 2024-09-13T12:58:15.080 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:14 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:58:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:14 smithi046 ceph-mon[32268]: pgmap v11: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 7.7 MiB/s wr, 844 op/s 2024-09-13T12:58:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:14 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:58:15.597 INFO:teuthology.orchestra.run.smithi042.stdout: 5 67443 12.62 MB/sec execute 20 sec latency 40.901 ms 2024-09-13T12:58:16.597 INFO:teuthology.orchestra.run.smithi042.stdout: 5 67914 12.77 MB/sec execute 21 sec latency 49.490 ms 2024-09-13T12:58:17.111 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:16 smithi042 ceph-mon[23723]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 623 op/s 2024-09-13T12:58:17.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:16 smithi046 ceph-mon[32268]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 623 op/s 2024-09-13T12:58:17.597 INFO:teuthology.orchestra.run.smithi042.stdout: 5 68272 12.51 MB/sec execute 22 sec latency 38.123 ms 2024-09-13T12:58:18.597 INFO:teuthology.orchestra.run.smithi042.stdout: 5 68784 12.29 MB/sec execute 23 sec latency 46.879 ms 2024-09-13T12:58:19.139 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:18 smithi042 ceph-mon[23723]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 552 op/s 2024-09-13T12:58:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:18 smithi046 ceph-mon[32268]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 552 op/s 2024-09-13T12:58:19.597 INFO:teuthology.orchestra.run.smithi042.stdout: 5 69313 12.49 MB/sec execute 24 sec latency 40.509 ms 2024-09-13T12:58:20.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 69995 12.61 MB/sec execute 25 sec latency 73.649 ms 2024-09-13T12:58:20.650 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:20 smithi042 ceph-mon[23723]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 853 op/s 2024-09-13T12:58:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:20 smithi046 ceph-mon[32268]: pgmap v14: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.1 MiB/s wr, 853 op/s 2024-09-13T12:58:21.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 70437 12.71 MB/sec execute 26 sec latency 42.591 ms 2024-09-13T12:58:22.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 70919 12.80 MB/sec execute 27 sec latency 44.633 ms 2024-09-13T12:58:22.929 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:22 smithi042 ceph-mon[23723]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 593 op/s 2024-09-13T12:58:23.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:22 smithi046 ceph-mon[32268]: pgmap v15: 97 pgs: 97 active+clean; 342 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.4 MiB/s wr, 593 op/s 2024-09-13T12:58:23.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 71549 13.12 MB/sec execute 28 sec latency 32.881 ms 2024-09-13T12:58:24.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 72030 12.92 MB/sec execute 29 sec latency 43.109 ms 2024-09-13T12:58:24.943 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:24 smithi042 ceph-mon[23723]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 813 op/s 2024-09-13T12:58:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:24 smithi046 ceph-mon[32268]: pgmap v16: 97 pgs: 97 active+clean; 344 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.5 MiB/s wr, 813 op/s 2024-09-13T12:58:25.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 72403 12.65 MB/sec execute 30 sec latency 39.227 ms 2024-09-13T12:58:26.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 72776 12.63 MB/sec execute 31 sec latency 41.670 ms 2024-09-13T12:58:26.999 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:26 smithi042 ceph-mon[23723]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-09-13T12:58:27.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:26 smithi046 ceph-mon[32268]: pgmap v17: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-09-13T12:58:27.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 73341 12.67 MB/sec execute 32 sec latency 43.083 ms 2024-09-13T12:58:28.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 73725 12.56 MB/sec execute 33 sec latency 44.834 ms 2024-09-13T12:58:29.028 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:28 smithi042 ceph-mon[23723]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-09-13T12:58:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:28 smithi046 ceph-mon[32268]: pgmap v18: 97 pgs: 97 active+clean; 347 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.9 KiB/s rd, 6.9 MiB/s wr, 621 op/s 2024-09-13T12:58:29.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 74077 12.56 MB/sec execute 34 sec latency 54.491 ms 2024-09-13T12:58:30.048 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:29 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:58:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:29 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:58:30.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 74410 12.55 MB/sec execute 35 sec latency 40.714 ms 2024-09-13T12:58:30.839 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:30 smithi042 ceph-mon[23723]: pgmap v19: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 874 op/s 2024-09-13T12:58:30.839 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:30 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:30.839 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:30 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:30 smithi046 ceph-mon[32268]: pgmap v19: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 9.1 MiB/s wr, 874 op/s 2024-09-13T12:58:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:30 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:30 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:31.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 74758 12.46 MB/sec execute 36 sec latency 52.276 ms 2024-09-13T12:58:31.858 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:31 smithi042 ceph-mon[23723]: Reconfiguring grafana.smithi042 (dependencies changed)... 2024-09-13T12:58:31.858 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:31 smithi042 ceph-mon[23723]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T12:58:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:31 smithi046 ceph-mon[32268]: Reconfiguring grafana.smithi042 (dependencies changed)... 2024-09-13T12:58:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:31 smithi046 ceph-mon[32268]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-13T12:58:32.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 75056 12.41 MB/sec execute 37 sec latency 59.260 ms 2024-09-13T12:58:32.852 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:32 smithi042 ceph-mon[23723]: pgmap v20: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 573 op/s 2024-09-13T12:58:32.852 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:32 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:32.852 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:32 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:32.852 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:32 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T12:58:32.852 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:32 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T12:58:32.852 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:32 smithi042 ceph-mon[23723]: Reconfiguring daemon grafana.smithi042 on smithi042 2024-09-13T12:58:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:32 smithi046 ceph-mon[32268]: pgmap v20: 97 pgs: 97 active+clean; 334 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 573 op/s 2024-09-13T12:58:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:32 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:32 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:32 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T12:58:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:32 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T12:58:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:32 smithi046 ceph-mon[32268]: Reconfiguring daemon grafana.smithi042 on smithi042 2024-09-13T12:58:33.607 INFO:teuthology.orchestra.run.smithi042.stdout: 5 75310 12.27 MB/sec execute 38 sec latency 58.025 ms 2024-09-13T12:58:34.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 75716 12.08 MB/sec execute 39 sec latency 39.410 ms 2024-09-13T12:58:34.951 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:34 smithi042 ceph-mon[23723]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 763 op/s 2024-09-13T12:58:35.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:34 smithi046 ceph-mon[32268]: pgmap v21: 97 pgs: 97 active+clean; 330 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.9 MiB/s wr, 763 op/s 2024-09-13T12:58:35.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 76073 11.92 MB/sec execute 40 sec latency 40.231 ms 2024-09-13T12:58:36.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 76472 12.00 MB/sec execute 41 sec latency 43.156 ms 2024-09-13T12:58:36.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:36 smithi042 ceph-mon[23723]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-09-13T12:58:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:36 smithi046 ceph-mon[32268]: pgmap v22: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.0 MiB/s wr, 555 op/s 2024-09-13T12:58:37.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 76939 11.97 MB/sec execute 42 sec latency 38.644 ms 2024-09-13T12:58:38.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 77425 12.01 MB/sec execute 43 sec latency 39.512 ms 2024-09-13T12:58:38.906 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:38 smithi042 ceph-mon[23723]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-13T12:58:38.906 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:38 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:38.906 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:38 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:39.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:38 smithi046 ceph-mon[32268]: pgmap v23: 97 pgs: 97 active+clean; 331 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 4.6 MiB/s wr, 455 op/s 2024-09-13T12:58:39.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:38 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:39.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:38 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:39.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 77879 12.13 MB/sec execute 44 sec latency 31.677 ms 2024-09-13T12:58:39.697 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:58:39.913 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:39 smithi042 ceph-mon[23723]: Reconfiguring node-exporter.smithi042 (dependencies changed)... 2024-09-13T12:58:39.913 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:39 smithi042 ceph-mon[23723]: Deploying daemon node-exporter.smithi042 on smithi042 2024-09-13T12:58:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:39 smithi046 ceph-mon[32268]: Reconfiguring node-exporter.smithi042 (dependencies changed)... 2024-09-13T12:58:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:39 smithi046 ceph-mon[32268]: Deploying daemon node-exporter.smithi042 on smithi042 2024-09-13T12:58:40.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 78426 12.19 MB/sec execute 45 sec latency 92.220 ms 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 starting - - - - 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (12m) 30s ago 12m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (8m) 35s ago 8m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 starting - - - - 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (3m) 30s ago 3m 37.4M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (3m) 35s ago 3m 14.2M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (13m) 30s ago 13m 231M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (65s) 35s ago 8m 495M - 19.3.0-4917-gb0201b8c 1641fe80ce72 5409c2ac218e 2024-09-13T12:58:40.769 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (13m) 30s ago 13m 51.7M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (8m) 35s ago 8m 38.6M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (2m) 30s ago 2m 235M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (2m) 35s ago 2m 75.7M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (9m) 30s ago 9m 19.1M - 1.3.1 1dbe0e931976 5a333e64c22b 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (8m) 35s ago 8m 18.9M - 1.3.1 1dbe0e931976 7086f6b8e52b 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (7m) 30s ago 7m 284M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (7m) 30s ago 7m 206M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (6m) 30s ago 6m 164M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (6m) 30s ago 6m 219M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (6m) 35s ago 6m 244M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (5m) 35s ago 5m 272M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (5m) 35s ago 5m 246M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (4m) 35s ago 4m 336M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:58:40.770 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (8m) 30s ago 8m 67.3M - 2.33.4 514e6a882f6e 147d67f2f5fb 2024-09-13T12:58:40.969 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:40 smithi042 ceph-mon[23723]: from='client.24471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:40.969 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:40 smithi042 ceph-mon[23723]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 701 op/s 2024-09-13T12:58:41.363 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:58:41.363 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:58:41.363 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:58:41.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:41.364 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:58:41.364 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T12:58:41.364 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T12:58:41.364 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:41.364 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:58:41.365 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:58:41.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:41.365 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:58:41.365 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:58:41.365 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:58:41.365 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:58:41.366 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T12:58:41.366 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T12:58:41.366 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:58:41.366 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:58:41.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:40 smithi046 ceph-mon[32268]: from='client.24471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:41.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:40 smithi046 ceph-mon[32268]: pgmap v24: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 6.9 MiB/s wr, 701 op/s 2024-09-13T12:58:41.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 78641 12.09 MB/sec execute 46 sec latency 62.459 ms 2024-09-13T12:58:41.883 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:58:41.883 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T12:58:41.883 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:58:41.883 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T12:58:41.884 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T12:58:41.884 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T12:58:41.884 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T12:58:41.884 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T12:58:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:58:42.039 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:41 smithi042 ceph-mon[23723]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:42.039 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:41 smithi042 ceph-mon[23723]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:42.039 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:41 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/2632764421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:58:42.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:41 smithi046 ceph-mon[32268]: from='client.24475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:42.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:41 smithi046 ceph-mon[32268]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:42.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:41 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/2632764421' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:58:42.516 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:58:42.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 78884 11.96 MB/sec execute 47 sec latency 41.591 ms 2024-09-13T12:58:43.050 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:42 smithi042 ceph-mon[23723]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 448 op/s 2024-09-13T12:58:43.050 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:42 smithi042 ceph-mon[23723]: from='client.14864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:43.050 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:42 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/3574585110' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:58:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:42 smithi046 ceph-mon[32268]: pgmap v25: 97 pgs: 97 active+clean; 331 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.7 MiB/s wr, 448 op/s 2024-09-13T12:58:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:42 smithi046 ceph-mon[32268]: from='client.14864 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:58:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:42 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3574585110' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:58:43.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 79284 11.81 MB/sec execute 48 sec latency 41.868 ms 2024-09-13T12:58:44.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 79660 11.71 MB/sec execute 49 sec latency 43.755 ms 2024-09-13T12:58:45.107 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:44 smithi042 ceph-mon[23723]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.1 MiB/s wr, 560 op/s 2024-09-13T12:58:45.107 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:44 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:58:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:44 smithi046 ceph-mon[32268]: pgmap v26: 97 pgs: 97 active+clean; 333 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.1 MiB/s wr, 560 op/s 2024-09-13T12:58:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:44 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:58:45.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 80031 11.76 MB/sec execute 50 sec latency 70.737 ms 2024-09-13T12:58:46.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 80657 11.82 MB/sec execute 51 sec latency 32.871 ms 2024-09-13T12:58:47.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:46 smithi042 ceph-mon[23723]: pgmap v27: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 484 op/s 2024-09-13T12:58:47.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:46 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:47.127 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:46 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:46 smithi046 ceph-mon[32268]: pgmap v27: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 5.0 MiB/s wr, 484 op/s 2024-09-13T12:58:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:46 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:46 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:58:47.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 81014 11.79 MB/sec execute 52 sec latency 63.341 ms 2024-09-13T12:58:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:47 smithi046 ceph-mon[32268]: Reconfiguring prometheus.smithi042 (dependencies changed)... 2024-09-13T12:58:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:47 smithi046 ceph-mon[32268]: Deploying daemon prometheus.smithi042 on smithi042 2024-09-13T12:58:48.145 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:47 smithi042 ceph-mon[23723]: Reconfiguring prometheus.smithi042 (dependencies changed)... 2024-09-13T12:58:48.145 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:47 smithi042 ceph-mon[23723]: Deploying daemon prometheus.smithi042 on smithi042 2024-09-13T12:58:48.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 81564 11.93 MB/sec execute 53 sec latency 57.304 ms 2024-09-13T12:58:49.152 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:48 smithi042 ceph-mon[23723]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 472 op/s 2024-09-13T12:58:49.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:48 smithi046 ceph-mon[32268]: pgmap v28: 97 pgs: 97 active+clean; 338 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.7 MiB/s wr, 472 op/s 2024-09-13T12:58:49.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 82141 12.04 MB/sec execute 54 sec latency 37.686 ms 2024-09-13T12:58:50.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 82369 11.95 MB/sec execute 55 sec latency 50.298 ms 2024-09-13T12:58:50.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:50 smithi042 ceph-mon[23723]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 746 op/s 2024-09-13T12:58:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:50 smithi046 ceph-mon[32268]: pgmap v29: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.6 MiB/s wr, 746 op/s 2024-09-13T12:58:51.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 82695 11.82 MB/sec execute 56 sec latency 40.192 ms 2024-09-13T12:58:52.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83045 11.67 MB/sec execute 57 sec latency 45.138 ms 2024-09-13T12:58:52.954 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:52 smithi042 ceph-mon[23723]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-09-13T12:58:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:52 smithi046 ceph-mon[32268]: pgmap v30: 97 pgs: 97 active+clean; 342 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 6.8 KiB/s rd, 5.4 MiB/s wr, 499 op/s 2024-09-13T12:58:53.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83353 11.63 MB/sec execute 58 sec latency 43.185 ms 2024-09-13T12:58:54.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83783 11.73 MB/sec execute 59 sec latency 36.837 ms 2024-09-13T12:58:55.034 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:54 smithi042 ceph-mon[23723]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 720 op/s 2024-09-13T12:58:55.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:54 smithi046 ceph-mon[32268]: pgmap v31: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 7.6 MiB/s wr, 720 op/s 2024-09-13T12:58:55.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 84233 11.66 MB/sec execute 60 sec latency 45.430 ms 2024-09-13T12:58:56.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 84651 11.65 MB/sec execute 61 sec latency 49.270 ms 2024-09-13T12:58:56.807 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:56 smithi042 ceph-mon[23723]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 607 op/s 2024-09-13T12:58:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:56 smithi046 ceph-mon[32268]: pgmap v32: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 6.1 MiB/s wr, 607 op/s 2024-09-13T12:58:57.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 85176 11.74 MB/sec execute 62 sec latency 35.229 ms 2024-09-13T12:58:58.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 85680 11.83 MB/sec execute 63 sec latency 27.895 ms 2024-09-13T12:58:58.837 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:58 smithi042 ceph-mon[23723]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 494 op/s 2024-09-13T12:58:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:58 smithi046 ceph-mon[32268]: pgmap v33: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.2 MiB/s wr, 494 op/s 2024-09-13T12:58:59.608 INFO:teuthology.orchestra.run.smithi042.stdout: 5 86070 11.82 MB/sec execute 64 sec latency 42.436 ms 2024-09-13T12:58:59.850 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:58:59 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:58:59 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:00.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 86494 11.69 MB/sec execute 65 sec latency 32.555 ms 2024-09-13T12:59:00.856 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:00 smithi042 ceph-mon[23723]: pgmap v34: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 760 op/s 2024-09-13T12:59:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:00 smithi046 ceph-mon[32268]: pgmap v34: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 8.0 MiB/s wr, 760 op/s 2024-09-13T12:59:01.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 87083 11.77 MB/sec execute 66 sec latency 24.115 ms 2024-09-13T12:59:02.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 87736 11.88 MB/sec execute 67 sec latency 25.089 ms 2024-09-13T12:59:02.867 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:02 smithi042 ceph-mon[23723]: pgmap v35: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 486 op/s 2024-09-13T12:59:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:02 smithi046 ceph-mon[32268]: pgmap v35: 97 pgs: 97 active+clean; 342 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 5.1 MiB/s wr, 486 op/s 2024-09-13T12:59:03.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 88266 11.86 MB/sec execute 68 sec latency 36.514 ms 2024-09-13T12:59:04.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 88895 11.99 MB/sec execute 69 sec latency 31.662 ms 2024-09-13T12:59:04.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:04 smithi042 ceph-mon[23723]: pgmap v36: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 803 op/s 2024-09-13T12:59:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:04 smithi046 ceph-mon[32268]: pgmap v36: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 8.1 MiB/s wr, 803 op/s 2024-09-13T12:59:05.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 89676 12.18 MB/sec execute 70 sec latency 35.484 ms 2024-09-13T12:59:06.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 90318 12.15 MB/sec execute 71 sec latency 35.799 ms 2024-09-13T12:59:06.903 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:06 smithi042 ceph-mon[23723]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 582 op/s 2024-09-13T12:59:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:06 smithi046 ceph-mon[32268]: pgmap v37: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 582 op/s 2024-09-13T12:59:07.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 90854 12.24 MB/sec execute 72 sec latency 30.437 ms 2024-09-13T12:59:08.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 91268 12.20 MB/sec execute 73 sec latency 63.649 ms 2024-09-13T12:59:08.939 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:08 smithi042 ceph-mon[23723]: pgmap v38: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 582 op/s 2024-09-13T12:59:09.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:08 smithi046 ceph-mon[32268]: pgmap v38: 97 pgs: 97 active+clean; 341 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.9 MiB/s wr, 582 op/s 2024-09-13T12:59:09.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 91636 12.14 MB/sec execute 74 sec latency 66.976 ms 2024-09-13T12:59:10.269 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:10.269 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:10 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:10.269 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:10 smithi042 ceph-mon[23723]: Reconfiguring node-exporter.smithi046 (dependencies changed)... 2024-09-13T12:59:10.269 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:10 smithi042 ceph-mon[23723]: Deploying daemon node-exporter.smithi046 on smithi046 2024-09-13T12:59:10.269 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:10 smithi042 ceph-mon[23723]: pgmap v39: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 928 op/s 2024-09-13T12:59:10.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:10.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:10 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:10.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:10 smithi046 ceph-mon[32268]: Reconfiguring node-exporter.smithi046 (dependencies changed)... 2024-09-13T12:59:10.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:10 smithi046 ceph-mon[32268]: Deploying daemon node-exporter.smithi046 on smithi046 2024-09-13T12:59:10.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:10 smithi046 ceph-mon[32268]: pgmap v39: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 9.4 MiB/s wr, 928 op/s 2024-09-13T12:59:10.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 91802 12.03 MB/sec execute 75 sec latency 56.015 ms 2024-09-13T12:59:11.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 92046 12.00 MB/sec execute 76 sec latency 88.083 ms 2024-09-13T12:59:12.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 92453 11.99 MB/sec execute 77 sec latency 64.008 ms 2024-09-13T12:59:12.849 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:12 smithi042 ceph-mon[23723]: pgmap v40: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 662 op/s 2024-09-13T12:59:13.004 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:59:13.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:12 smithi046 ceph-mon[32268]: pgmap v40: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.6 MiB/s wr, 662 op/s 2024-09-13T12:59:13.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 92745 12.00 MB/sec execute 78 sec latency 56.701 ms 2024-09-13T12:59:13.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:13 smithi042 ceph-mon[23723]: from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:13.915 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:59:13.915 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 starting - - - - 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (12m) 63s ago 12m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (9m) 68s ago 9m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 starting - - - - 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (3m) 63s ago 3m 37.4M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (3m) 68s ago 3m 14.2M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:9283 running (14m) 63s ago 14m 231M - 17.2.0 e1d6a67b021e 95db3e31813b 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (98s) 68s ago 9m 495M - 19.3.0-4917-gb0201b8c 1641fe80ce72 5409c2ac218e 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (14m) 63s ago 14m 51.7M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (9m) 68s ago 9m 38.6M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (3m) 63s ago 3m 235M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (3m) 68s ago 3m 75.7M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:59:13.916 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 starting - - - - 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (9m) 68s ago 9m 18.9M - 1.3.1 1dbe0e931976 7086f6b8e52b 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (8m) 63s ago 8m 284M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (7m) 63s ago 7m 206M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (7m) 63s ago 7m 164M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (7m) 63s ago 7m 219M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (6m) 68s ago 6m 244M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (6m) 68s ago 6m 272M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (5m) 68s ago 5m 246M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (5m) 68s ago 5m 336M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:59:13.917 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 starting - - - - 2024-09-13T12:59:14.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:13 smithi046 ceph-mon[32268]: from='client.14872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:14.463 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:59:14.464 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:59:14.609 INFO:teuthology.orchestra.run.smithi042.stdout: 5 93016 11.95 MB/sec execute 79 sec latency 68.199 ms 2024-09-13T12:59:14.916 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:14 smithi042 ceph-mon[23723]: from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:14.916 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:14 smithi042 ceph-mon[23723]: pgmap v41: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.6 MiB/s wr, 837 op/s 2024-09-13T12:59:14.916 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:14 smithi042 ceph-mon[23723]: from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:14.916 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:14 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:14.916 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:14 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/985990077' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:59:14.937 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:14 smithi046 ceph-mon[32268]: from='client.14876 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:14.938 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:14 smithi046 ceph-mon[32268]: pgmap v41: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 8.6 MiB/s wr, 837 op/s 2024-09-13T12:59:14.938 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:14 smithi046 ceph-mon[32268]: from='client.14880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:14.938 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:14 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:14.938 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:14 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/985990077' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:59:14.938 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:59:14.938 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T12:59:14.938 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:59:14.938 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T12:59:14.938 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T12:59:14.939 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "1/23 daemons upgraded", 2024-09-13T12:59:14.939 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T12:59:14.939 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T12:59:14.939 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:59:15.481 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:59:15.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 93292 11.86 MB/sec execute 80 sec latency 56.652 ms 2024-09-13T12:59:15.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='client.14888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:15.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:15.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:15.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi042.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi042.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:15.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:15 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/132652271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:59:16.087 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='client.14888 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:16.087 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:16.087 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:16.087 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:59:16.087 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi042.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi042.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.088 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:15 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/132652271' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:59:16.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 93665 11.75 MB/sec execute 81 sec latency 64.392 ms 2024-09-13T12:59:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:59:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T12:59:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi042.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi042.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: pgmap v42: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 521 op/s 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: Upgrade: Need to upgrade myself (mgr.smithi046.hcgddm) 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: Upgrade: Need to upgrade myself (mgr.smithi046.hcgddm) 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:16 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.939 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi042.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi042.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi042.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: pgmap v42: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 521 op/s 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: Upgrade: Need to upgrade myself (mgr.smithi046.hcgddm) 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: Upgrade: Need to upgrade myself (mgr.smithi046.hcgddm) 2024-09-13T12:59:16.940 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.941 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:16.941 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:16 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:17.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 93944 11.71 MB/sec execute 82 sec latency 59.158 ms 2024-09-13T12:59:17.838 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:17 smithi042 ceph-mon[23723]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:18.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:17 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:18.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 94328 11.76 MB/sec execute 83 sec latency 46.847 ms 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: pgmap v43: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 521 op/s 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: Upgrade: Updating mgr.smithi042.jhsfyp 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:59:19.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:19 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: pgmap v43: 97 pgs: 97 active+clean; 336 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 8.4 KiB/s rd, 5.6 MiB/s wr, 521 op/s 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: Upgrade: Updating mgr.smithi042.jhsfyp 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T12:59:19.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:19 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:59:19.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 94720 11.73 MB/sec execute 84 sec latency 68.536 ms 2024-09-13T12:59:20.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:20 smithi042 ceph-mon[23723]: Deploying daemon mgr.smithi042.jhsfyp on smithi042 2024-09-13T12:59:20.350 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:20 smithi046 ceph-mon[32268]: Deploying daemon mgr.smithi042.jhsfyp on smithi042 2024-09-13T12:59:20.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 95110 11.68 MB/sec execute 85 sec latency 62.863 ms 2024-09-13T12:59:21.032 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:21 smithi042 ceph-mon[23723]: pgmap v44: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 725 op/s 2024-09-13T12:59:21.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:21 smithi046 ceph-mon[32268]: pgmap v44: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 12 KiB/s rd, 7.6 MiB/s wr, 725 op/s 2024-09-13T12:59:21.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 95302 11.60 MB/sec execute 86 sec latency 75.890 ms 2024-09-13T12:59:22.310 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:22 smithi042 ceph-mon[23723]: pgmap v45: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 379 op/s 2024-09-13T12:59:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:22 smithi046 ceph-mon[32268]: pgmap v45: 97 pgs: 97 active+clean; 336 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.1 KiB/s rd, 4.0 MiB/s wr, 379 op/s 2024-09-13T12:59:22.610 INFO:teuthology.orchestra.run.smithi042.stdout: 5 95632 11.58 MB/sec execute 87 sec latency 64.919 ms 2024-09-13T12:59:23.611 INFO:teuthology.orchestra.run.smithi042.stdout: 5 95860 11.54 MB/sec execute 88 sec latency 54.442 ms 2024-09-13T12:59:24.611 INFO:teuthology.orchestra.run.smithi042.stdout: 5 96099 11.48 MB/sec execute 89 sec latency 48.140 ms 2024-09-13T12:59:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:24 smithi046 ceph-mon[32268]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 556 op/s 2024-09-13T12:59:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:24 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:24 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:24 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:25.121 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:24 smithi042 ceph-mon[23723]: pgmap v46: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 556 op/s 2024-09-13T12:59:25.121 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:24 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:25.121 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:24 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:25.122 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:24 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:25.611 INFO:teuthology.orchestra.run.smithi042.stdout: 5 96428 11.51 MB/sec execute 90 sec latency 61.496 ms 2024-09-13T12:59:26.611 INFO:teuthology.orchestra.run.smithi042.stdout: 5 96777 11.47 MB/sec execute 91 sec latency 44.683 ms 2024-09-13T12:59:27.140 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:26 smithi042 ceph-mon[23723]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 380 op/s 2024-09-13T12:59:27.271 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:26 smithi046 ceph-mon[32268]: pgmap v47: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 380 op/s 2024-09-13T12:59:27.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 97114 11.37 MB/sec execute 92 sec latency 41.252 ms 2024-09-13T12:59:28.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 97393 11.33 MB/sec execute 93 sec latency 52.143 ms 2024-09-13T12:59:28.684 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:28 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:28.684 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:28 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:28.684 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:28 smithi042 ceph-mon[23723]: pgmap v48: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 380 op/s 2024-09-13T12:59:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:28 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:28 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:28 smithi046 ceph-mon[32268]: pgmap v48: 97 pgs: 97 active+clean; 341 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.8 MiB/s wr, 380 op/s 2024-09-13T12:59:29.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 97661 11.31 MB/sec execute 94 sec latency 42.813 ms 2024-09-13T12:59:29.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:29 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:29 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:30.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 98062 11.36 MB/sec execute 95 sec latency 39.629 ms 2024-09-13T12:59:30.715 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:30 smithi042 ceph-mon[23723]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 565 op/s 2024-09-13T12:59:30.715 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:30 smithi042 ceph-mon[23723]: mgrmap e28: smithi046.hcgddm(active, since 93s), standbys: smithi042.jhsfyp 2024-09-13T12:59:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:30 smithi046 ceph-mon[32268]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.6 MiB/s wr, 565 op/s 2024-09-13T12:59:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:30 smithi046 ceph-mon[32268]: mgrmap e28: smithi046.hcgddm(active, since 93s), standbys: smithi042.jhsfyp 2024-09-13T12:59:31.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 98421 11.31 MB/sec execute 96 sec latency 59.657 ms 2024-09-13T12:59:31.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:31 smithi042 ceph-mon[23723]: Standby manager daemon smithi042.jhsfyp restarted 2024-09-13T12:59:31.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:31 smithi042 ceph-mon[23723]: Standby manager daemon smithi042.jhsfyp started 2024-09-13T12:59:31.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:31 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/crt"}]: dispatch 2024-09-13T12:59:31.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:31 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:59:31.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:31 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/key"}]: dispatch 2024-09-13T12:59:31.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:31 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:59:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:31 smithi046 ceph-mon[32268]: Standby manager daemon smithi042.jhsfyp restarted 2024-09-13T12:59:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:31 smithi046 ceph-mon[32268]: Standby manager daemon smithi042.jhsfyp started 2024-09-13T12:59:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:31 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/crt"}]: dispatch 2024-09-13T12:59:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:31 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:59:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:31 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi042.jhsfyp/key"}]: dispatch 2024-09-13T12:59:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:31 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:59:32.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 98719 11.24 MB/sec execute 97 sec latency 69.532 ms 2024-09-13T12:59:32.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:32 smithi042 ceph-mon[23723]: mgrmap e29: smithi046.hcgddm(active, since 94s), standbys: smithi042.jhsfyp 2024-09-13T12:59:32.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:32 smithi042 ceph-mon[23723]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T12:59:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:32 smithi046 ceph-mon[32268]: mgrmap e29: smithi046.hcgddm(active, since 94s), standbys: smithi042.jhsfyp 2024-09-13T12:59:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:32 smithi046 ceph-mon[32268]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T12:59:33.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 98924 11.16 MB/sec execute 98 sec latency 65.871 ms 2024-09-13T12:59:33.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:33 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:33.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:33 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:33.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:33 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:33.790 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:33 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:34.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:33 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:34.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:33 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:34.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:33 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:34.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:33 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:34.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 99301 11.20 MB/sec execute 99 sec latency 44.488 ms 2024-09-13T12:59:34.811 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:34 smithi042 ceph-mon[23723]: pgmap v51: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 554 op/s 2024-09-13T12:59:35.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:34 smithi046 ceph-mon[32268]: pgmap v51: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.5 MiB/s wr, 554 op/s 2024-09-13T12:59:35.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 99833 11.28 MB/sec execute 100 sec latency 29.037 ms 2024-09-13T12:59:36.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 100293 11.27 MB/sec execute 101 sec latency 30.813 ms 2024-09-13T12:59:36.951 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:36 smithi042 ceph-mon[23723]: pgmap v52: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-09-13T12:59:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:36 smithi046 ceph-mon[32268]: pgmap v52: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-09-13T12:59:37.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 100716 11.21 MB/sec execute 102 sec latency 43.694 ms 2024-09-13T12:59:38.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 101005 11.17 MB/sec execute 103 sec latency 118.309 ms 2024-09-13T12:59:38.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:38 smithi042 ceph-mon[23723]: pgmap v53: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-09-13T12:59:39.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:38 smithi046 ceph-mon[32268]: pgmap v53: 97 pgs: 97 active+clean; 343 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 377 op/s 2024-09-13T12:59:39.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 101341 11.17 MB/sec execute 104 sec latency 61.704 ms 2024-09-13T12:59:40.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 101754 11.21 MB/sec execute 105 sec latency 80.540 ms 2024-09-13T12:59:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:59:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:59:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: pgmap v54: 97 pgs: 97 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.3 MiB/s wr, 629 op/s 2024-09-13T12:59:40.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:40.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr fail", "who": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:59:40.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr fail", "who": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:59:40.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:40 smithi046 ceph-mon[32268]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T12:59:40.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: pgmap v54: 97 pgs: 97 active+clean; 347 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 KiB/s rd, 6.3 MiB/s wr, 629 op/s 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 172.21.15.46:0/1012179786' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr fail", "who": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "mgr fail", "who": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:59:40.799 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:40 smithi042 ceph-mon[23723]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T12:59:41.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 102204 11.16 MB/sec execute 106 sec latency 41.977 ms 2024-09-13T12:59:41.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd='[{"prefix": "mgr fail", "who": "smithi046.hcgddm"}]': finished 2024-09-13T12:59:41.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: mgrmap e30: smithi042.jhsfyp(active, starting, since 0.535542s) 2024-09-13T12:59:41.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:59:41.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:59:41.820 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:59:41.821 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:59:41.821 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:41 smithi042 ceph-mon[23723]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.14798 ' entity='mgr.smithi046.hcgddm' cmd='[{"prefix": "mgr fail", "who": "smithi046.hcgddm"}]': finished 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: mgrmap e30: smithi042.jhsfyp(active, starting, since 0.535542s) 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"}]: dispatch 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"}]: dispatch 2024-09-13T12:59:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T12:59:41.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:41 smithi046 ceph-mon[32268]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T12:59:42.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 102383 11.10 MB/sec execute 107 sec latency 71.597 ms 2024-09-13T12:59:42.831 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: mgrmap e31: smithi042.jhsfyp(active, since 1.54282s) 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:59:42.832 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:42 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: mgrmap e31: smithi042.jhsfyp(active, since 1.54282s) 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: pgmap v3: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T12:59:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"}]: dispatch 2024-09-13T12:59:43.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:42 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"}]: dispatch 2024-09-13T12:59:43.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 102771 11.13 MB/sec execute 108 sec latency 55.587 ms 2024-09-13T12:59:43.859 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:43 smithi042 ceph-mon[23723]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T12:59:43.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:43 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:43.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:43 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:44.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:43 smithi046 ceph-mon[32268]: pgmap v4: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T12:59:44.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:43 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:44.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:43 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:44.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 103111 11.11 MB/sec execute 109 sec latency 45.813 ms 2024-09-13T12:59:44.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:44 smithi042 ceph-mon[23723]: mgrmap e32: smithi042.jhsfyp(active, since 3s) 2024-09-13T12:59:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:44 smithi046 ceph-mon[32268]: mgrmap e32: smithi042.jhsfyp(active, since 3s) 2024-09-13T12:59:45.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 103412 11.14 MB/sec execute 110 sec latency 52.973 ms 2024-09-13T12:59:45.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:45 smithi042 ceph-mon[23723]: [13/Sep/2024:12:59:43] ENGINE Bus STARTING 2024-09-13T12:59:45.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:45 smithi042 ceph-mon[23723]: [13/Sep/2024:12:59:44] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T12:59:45.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:45 smithi042 ceph-mon[23723]: [13/Sep/2024:12:59:44] ENGINE Error in HTTPServer.serve 2024-09-13T12:59:45.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Traceback (most recent call last): 2024-09-13T12:59:45.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._run(expiration_interval) 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T12:59:45.911 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self.do_handshake() 2024-09-13T12:59:45.912 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T12:59:45.912 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T12:59:45.912 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T12:59:45.912 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:45 smithi042 ceph-mon[23723]: [13/Sep/2024:12:59:44] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T12:59:45.912 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:45 smithi042 ceph-mon[23723]: [13/Sep/2024:12:59:44] ENGINE Bus STARTED 2024-09-13T12:59:45.912 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:45 smithi042 ceph-mon[23723]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T12:59:46.057 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:45 smithi046 ceph-mon[32268]: [13/Sep/2024:12:59:43] ENGINE Bus STARTING 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:45 smithi046 ceph-mon[32268]: [13/Sep/2024:12:59:44] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:45 smithi046 ceph-mon[32268]: [13/Sep/2024:12:59:44] ENGINE Error in HTTPServer.serve 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Traceback (most recent call last): 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._connections.run(self.expiration_interval) 2024-09-13T12:59:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._run(expiration_interval) 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: s = self.context.wrap_socket( 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: return self.sslsocket_class._create( 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self.do_handshake() 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T12:59:46.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._sslobj.do_handshake() 2024-09-13T12:59:46.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T12:59:46.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:45 smithi046 ceph-mon[32268]: [13/Sep/2024:12:59:44] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T12:59:46.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:45 smithi046 ceph-mon[32268]: [13/Sep/2024:12:59:44] ENGINE Bus STARTED 2024-09-13T12:59:46.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:45 smithi046 ceph-mon[32268]: pgmap v5: 97 pgs: 97 active+clean; 339 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T12:59:46.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 103670 11.09 MB/sec execute 111 sec latency 95.611 ms 2024-09-13T12:59:46.671 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:46 smithi042 ceph-mon[23723]: mgrmap e33: smithi042.jhsfyp(active, since 5s) 2024-09-13T12:59:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:46 smithi046 ceph-mon[32268]: mgrmap e33: smithi042.jhsfyp(active, since 5s) 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (77s) 14s ago 13m 19.7M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (13m) 14s ago 13m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (9m) 0s ago 9m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (69s) 14s ago 10m 43.4M - dad864ee21e9 7144014c89b8 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (4m) 14s ago 4m 39.0M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (4m) 0s ago 4m 14.7M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T12:59:47.135 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (23s) 14s ago 14m 66.2M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283 running (2m) 0s ago 9m 177M - 19.3.0-4917-gb0201b8c 1641fe80ce72 5409c2ac218e 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (14m) 14s ago 14m 54.3M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (9m) 0s ago 9m 45.1M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (4m) 14s ago 4m 240M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (4m) 0s ago 3m 83.8M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (61s) 14s ago 10m 8392k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (32s) 0s ago 9m 9675k - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (9m) 14s ago 9m 384M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (8m) 14s ago 8m 288M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (8m) 14s ago 8m 230M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (7m) 14s ago 7m 354M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (7m) 0s ago 7m 371M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T12:59:47.136 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (6m) 0s ago 6m 397M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T12:59:47.137 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (6m) 0s ago 6m 335M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T12:59:47.137 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (5m) 0s ago 5m 497M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T12:59:47.137 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (39s) 14s ago 9m 41.8M - 2.51.0 1d3b7f56885b 6667eedc9960 2024-09-13T12:59:47.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 104030 11.04 MB/sec execute 112 sec latency 36.500 ms 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='client.14926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='client.14930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: pgmap v6: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 927 KiB/s wr, 33 op/s 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: Standby manager daemon smithi046.hcgddm started 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: mgrmap e34: smithi042.jhsfyp(active, since 6s), standbys: smithi046.hcgddm 2024-09-13T12:59:47.679 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:59:47.680 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:47.680 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:47.680 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:47.680 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:47 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T12:59:47.878 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T12:59:47.879 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='client.14926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='client.14930 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: pgmap v6: 97 pgs: 97 active+clean; 340 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 927 KiB/s wr, 33 op/s 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm started 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T12:59:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/1138030379' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: mgrmap e34: smithi042.jhsfyp(active, since 6s), standbys: smithi046.hcgddm 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"}]: dispatch 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:48.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:47 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:48.538 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T12:59:48.538 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T12:59:48.538 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T12:59:48.539 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T12:59:48.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 104616 11.04 MB/sec execute 113 sec latency 26.341 ms 2024-09-13T12:59:48.687 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:48 smithi042 ceph-mon[23723]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:48.687 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:48 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/688653381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:59:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:48 smithi046 ceph-mon[32268]: from='client.14934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:48 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/688653381' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T12:59:49.313 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T12:59:49.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 105347 11.17 MB/sec execute 114 sec latency 28.463 ms 2024-09-13T12:59:49.691 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:49 smithi042 ceph-mon[23723]: from='client.24549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:49.691 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:49 smithi042 ceph-mon[23723]: pgmap v7: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.9 MiB/s wr, 336 op/s 2024-09-13T12:59:49.691 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:49 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/893103346' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:59:50.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:49 smithi046 ceph-mon[32268]: from='client.24549 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T12:59:50.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:49 smithi046 ceph-mon[32268]: pgmap v7: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 30 KiB/s rd, 3.9 MiB/s wr, 336 op/s 2024-09-13T12:59:50.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:49 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/893103346' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T12:59:50.612 INFO:teuthology.orchestra.run.smithi042.stdout: 5 105748 11.12 MB/sec execute 115 sec latency 70.461 ms 2024-09-13T12:59:51.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 105999 11.07 MB/sec execute 116 sec latency 54.769 ms 2024-09-13T12:59:51.740 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:51 smithi042 ceph-mon[23723]: pgmap v8: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.1 MiB/s wr, 261 op/s 2024-09-13T12:59:51.740 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:51 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:51.740 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:51 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:52.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:51 smithi046 ceph-mon[32268]: pgmap v8: 97 pgs: 97 active+clean; 335 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.1 MiB/s wr, 261 op/s 2024-09-13T12:59:52.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:51 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:52.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:51 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:52.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 106335 11.09 MB/sec execute 117 sec latency 40.813 ms 2024-09-13T12:59:52.750 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:52 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:52.750 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:52 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:52 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:52 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:53.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 106745 11.09 MB/sec execute 118 sec latency 145.109 ms 2024-09-13T12:59:53.762 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:53 smithi042 ceph-mon[23723]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.3 MiB/s wr, 513 op/s 2024-09-13T12:59:54.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:53 smithi046 ceph-mon[32268]: pgmap v9: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 5.3 MiB/s wr, 513 op/s 2024-09-13T12:59:54.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 107104 11.12 MB/sec execute 119 sec latency 151.805 ms 2024-09-13T12:59:55.551 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:55 smithi042 ceph-mon[23723]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.9 MiB/s wr, 469 op/s 2024-09-13T12:59:55.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 107382 11.09 MB/sec execute 120 sec latency 58.519 ms 2024-09-13T12:59:55.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:55 smithi046 ceph-mon[32268]: pgmap v10: 97 pgs: 97 active+clean; 338 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 4.9 MiB/s wr, 469 op/s 2024-09-13T12:59:56.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 107661 11.05 MB/sec execute 121 sec latency 42.514 ms 2024-09-13T12:59:56.825 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:56 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:56 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T12:59:57.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 107946 10.98 MB/sec execute 122 sec latency 82.777 ms 2024-09-13T12:59:57.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:57 smithi042 ceph-mon[23723]: pgmap v11: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.1 MiB/s wr, 626 op/s 2024-09-13T12:59:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:57 smithi046 ceph-mon[32268]: pgmap v11: 97 pgs: 97 active+clean; 343 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.1 MiB/s wr, 626 op/s 2024-09-13T12:59:58.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 108234 10.96 MB/sec execute 123 sec latency 39.864 ms 2024-09-13T12:59:58.897 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.897 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.899 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.899 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.899 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:58.899 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: Reconfiguring prometheus.smithi042 (dependencies changed)... 2024-09-13T12:59:58.899 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 12:59:58 smithi042 ceph-mon[23723]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.7 MiB/s wr, 654 op/s 2024-09-13T12:59:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T12:59:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T12:59:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T12:59:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: Reconfiguring prometheus.smithi042 (dependencies changed)... 2024-09-13T12:59:59.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 12:59:58 smithi046 ceph-mon[32268]: pgmap v12: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 6.7 MiB/s wr, 654 op/s 2024-09-13T12:59:59.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 108482 10.94 MB/sec execute 124 sec latency 59.402 ms 2024-09-13T13:00:00.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 108748 10.94 MB/sec execute 125 sec latency 104.229 ms 2024-09-13T13:00:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:00 smithi046 ceph-mon[32268]: Reconfiguring daemon prometheus.smithi042 on smithi042 2024-09-13T13:00:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:00 smithi046 ceph-mon[32268]: overall HEALTH_OK 2024-09-13T13:00:00.704 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:00 smithi042 ceph-mon[23723]: Reconfiguring daemon prometheus.smithi042 on smithi042 2024-09-13T13:00:00.705 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:00 smithi042 ceph-mon[23723]: overall HEALTH_OK 2024-09-13T13:00:01.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 108956 10.87 MB/sec execute 126 sec latency 70.877 ms 2024-09-13T13:00:01.718 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:01 smithi042 ceph-mon[23723]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-13T13:00:01.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:01 smithi046 ceph-mon[32268]: pgmap v13: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.8 MiB/s wr, 473 op/s 2024-09-13T13:00:02.613 INFO:teuthology.orchestra.run.smithi042.stdout: 5 109238 10.82 MB/sec execute 127 sec latency 153.977 ms 2024-09-13T13:00:03.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 109407 10.75 MB/sec execute 128 sec latency 149.149 ms 2024-09-13T13:00:03.873 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:03 smithi042 ceph-mon[23723]: pgmap v14: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 640 op/s 2024-09-13T13:00:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:03 smithi046 ceph-mon[32268]: pgmap v14: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 6.4 MiB/s wr, 640 op/s 2024-09-13T13:00:04.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 109525 10.69 MB/sec execute 129 sec latency 142.056 ms 2024-09-13T13:00:05.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 109634 10.64 MB/sec execute 130 sec latency 76.021 ms 2024-09-13T13:00:05.734 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:05 smithi042 ceph-mon[23723]: pgmap v15: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 366 op/s 2024-09-13T13:00:05.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:05 smithi046 ceph-mon[32268]: pgmap v15: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.8 MiB/s wr, 366 op/s 2024-09-13T13:00:06.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 109793 10.59 MB/sec execute 131 sec latency 136.705 ms 2024-09-13T13:00:07.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:07 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:07.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:07 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:07.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:07 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T13:00:07.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:07 smithi042 ceph-mon[23723]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T13:00:07.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:07 smithi042 ceph-mon[23723]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.5 MiB/s wr, 456 op/s 2024-09-13T13:00:07.539 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:07 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T13:00:07.559 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:07 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:07.559 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:07 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:07.559 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:07 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T13:00:07.559 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:07 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T13:00:07.559 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:07 smithi046 ceph-mon[32268]: pgmap v16: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.5 MiB/s wr, 456 op/s 2024-09-13T13:00:07.559 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:07 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T13:00:07.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 110298 10.64 MB/sec execute 132 sec latency 71.503 ms 2024-09-13T13:00:08.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 110777 10.68 MB/sec execute 133 sec latency 33.130 ms 2024-09-13T13:00:08.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:08 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T13:00:08.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:08 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T13:00:08.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:08 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T13:00:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:08 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T13:00:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:08 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T13:00:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:08 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T13:00:09.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 110976 10.65 MB/sec execute 134 sec latency 61.758 ms 2024-09-13T13:00:09.831 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:09 smithi042 ceph-mon[23723]: Upgrade: Updating mgr.smithi046.hcgddm 2024-09-13T13:00:09.831 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:09 smithi042 ceph-mon[23723]: Deploying daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T13:00:09.831 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:09 smithi042 ceph-mon[23723]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 317 op/s 2024-09-13T13:00:09.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:09 smithi046 ceph-mon[32268]: Upgrade: Updating mgr.smithi046.hcgddm 2024-09-13T13:00:09.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:09 smithi046 ceph-mon[32268]: Deploying daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T13:00:09.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:09 smithi046 ceph-mon[32268]: pgmap v17: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 3.6 MiB/s wr, 317 op/s 2024-09-13T13:00:10.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 111269 10.61 MB/sec execute 135 sec latency 120.714 ms 2024-09-13T13:00:11.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 111512 10.56 MB/sec execute 136 sec latency 81.125 ms 2024-09-13T13:00:11.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:11 smithi042 ceph-mon[23723]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.6 MiB/s wr, 275 op/s 2024-09-13T13:00:11.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:11 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T13:00:11.859 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:11 smithi046 ceph-mon[32268]: pgmap v18: 97 pgs: 97 active+clean; 336 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.6 MiB/s wr, 275 op/s 2024-09-13T13:00:11.859 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:11 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T13:00:12.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 111703 10.51 MB/sec execute 137 sec latency 58.332 ms 2024-09-13T13:00:13.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 111921 10.51 MB/sec execute 138 sec latency 63.522 ms 2024-09-13T13:00:13.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:13 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:13.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:13 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:13.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:13 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T13:00:13.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:13 smithi042 ceph-mon[23723]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-09-13T13:00:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:13 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:13 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:13 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T13:00:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:13 smithi046 ceph-mon[32268]: pgmap v19: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.7 MiB/s wr, 469 op/s 2024-09-13T13:00:14.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 112111 10.46 MB/sec execute 139 sec latency 78.696 ms 2024-09-13T13:00:15.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 112365 10.46 MB/sec execute 140 sec latency 139.429 ms 2024-09-13T13:00:15.681 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:15 smithi042 ceph-mon[23723]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.2 MiB/s wr, 302 op/s 2024-09-13T13:00:15.767 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:15 smithi046 ceph-mon[32268]: pgmap v20: 97 pgs: 97 active+clean; 339 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 3.2 MiB/s wr, 302 op/s 2024-09-13T13:00:16.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 112460 10.39 MB/sec execute 141 sec latency 198.944 ms 2024-09-13T13:00:17.564 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:17 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.564 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:17 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.565 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:17 smithi042 ceph-mon[23723]: pgmap v21: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.1 MiB/s wr, 422 op/s 2024-09-13T13:00:17.565 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:17 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.565 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:17 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 112623 10.34 MB/sec execute 142 sec latency 183.989 ms 2024-09-13T13:00:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:17 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:17 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:17 smithi046 ceph-mon[32268]: pgmap v21: 97 pgs: 97 active+clean; 338 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.1 MiB/s wr, 422 op/s 2024-09-13T13:00:17.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:17 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:17.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:17 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:18.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:18 smithi042 ceph-mon[23723]: Standby manager daemon smithi046.hcgddm restarted 2024-09-13T13:00:18.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:18 smithi042 ceph-mon[23723]: Standby manager daemon smithi046.hcgddm started 2024-09-13T13:00:18.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:18 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T13:00:18.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:18 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T13:00:18.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:18 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T13:00:18.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:18 smithi042 ceph-mon[23723]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T13:00:18.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 112969 10.30 MB/sec execute 143 sec latency 99.089 ms 2024-09-13T13:00:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:18 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm restarted 2024-09-13T13:00:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:18 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm started 2024-09-13T13:00:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:18 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T13:00:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:18 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T13:00:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:18 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T13:00:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:18 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3333935823' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T13:00:19.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 113226 10.27 MB/sec execute 144 sec latency 88.736 ms 2024-09-13T13:00:19.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:19 smithi042 ceph-mon[23723]: mgrmap e35: smithi042.jhsfyp(active, since 38s), standbys: smithi046.hcgddm 2024-09-13T13:00:19.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:19 smithi042 ceph-mon[23723]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 346 op/s 2024-09-13T13:00:19.669 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:19 smithi046 ceph-mon[32268]: mgrmap e35: smithi042.jhsfyp(active, since 38s), standbys: smithi046.hcgddm 2024-09-13T13:00:19.669 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:19 smithi046 ceph-mon[32268]: pgmap v22: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 346 op/s 2024-09-13T13:00:19.743 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:00:20.585 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:00:20.585 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (111s) 0s ago 13m 19.7M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:00:20.586 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (13m) 0s ago 13m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T13:00:20.586 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (10m) 4s ago 10m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T13:00:20.586 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (103s) 0s ago 10m 44.0M - dad864ee21e9 7144014c89b8 2024-09-13T13:00:20.586 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (5m) 0s ago 5m 39.6M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:00:20.586 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (4m) 4s ago 4m 14.8M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (57s) 0s ago 15m 507M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (8s) 4s ago 10m 64.3M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (15m) 0s ago 15m 57.7M 2048M 17.2.0 e1d6a67b021e f35ac5b07f4e 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (10m) 4s ago 10m 46.5M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (4m) 0s ago 4m 258M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (4m) 4s ago 4m 86.0M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:00:20.587 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (94s) 0s ago 10m 10.0M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (65s) 4s ago 10m 9903k - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (9m) 0s ago 9m 447M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (9m) 0s ago 9m 345M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (8m) 0s ago 8m 260M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (8m) 0s ago 8m 401M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (7m) 4s ago 7m 388M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (7m) 4s ago 7m 434M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:00:20.588 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (6m) 4s ago 6m 368M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:00:20.589 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (6m) 4s ago 6m 523M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:00:20.589 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (14s) 0s ago 10m 43.9M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:00:20.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 113369 10.25 MB/sec execute 145 sec latency 110.791 ms 2024-09-13T13:00:20.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:20 smithi042 ceph-mon[23723]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:20.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:20 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:20.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:20 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:20 smithi046 ceph-mon[32268]: from='client.24561 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:20 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:20 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.192 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:00:21.193 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:21.194 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:00:21.194 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-09-13T13:00:21.194 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:00:21.194 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:00:21.194 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:00:21.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 113518 10.21 MB/sec execute 146 sec latency 170.873 ms 2024-09-13T13:00:21.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 327 op/s 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/1522977492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T13:00:21.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T13:00:21.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi042.jhsfyp"}]: dispatch 2024-09-13T13:00:21.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi042.jhsfyp"}]': finished 2024-09-13T13:00:21.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi046.hcgddm"}]: dispatch 2024-09-13T13:00:21.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi046.hcgddm"}]': finished 2024-09-13T13:00:21.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:21 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T13:00:21.711 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:00:21.712 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "2/23 daemons upgraded", 2024-09-13T13:00:21.712 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T13:00:21.712 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:00:21.712 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:00:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='client.24567 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: pgmap v23: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.3 MiB/s wr, 327 op/s 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1522977492' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi042.jhsfyp"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi042.jhsfyp"}]': finished 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi046.hcgddm"}]: dispatch 2024-09-13T13:00:21.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.smithi046.hcgddm"}]': finished 2024-09-13T13:00:21.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:21 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T13:00:22.207 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:00:22.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 113775 10.20 MB/sec execute 147 sec latency 67.884 ms 2024-09-13T13:00:22.709 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:22 smithi042 ceph-mon[23723]: Upgrade: Setting container_image for all mgr 2024-09-13T13:00:22.709 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:22 smithi042 ceph-mon[23723]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:22.709 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:22 smithi042 ceph-mon[23723]: from='client.? 172.21.15.42:0/615668443' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T13:00:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:22 smithi046 ceph-mon[32268]: Upgrade: Setting container_image for all mgr 2024-09-13T13:00:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:22 smithi046 ceph-mon[32268]: from='client.14970 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:22 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/615668443' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T13:00:23.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 114405 10.27 MB/sec execute 148 sec latency 19.041 ms 2024-09-13T13:00:23.713 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:23 smithi042 ceph-mon[23723]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-09-13T13:00:23.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:23 smithi046 ceph-mon[32268]: pgmap v24: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.4 KiB/s rd, 4.6 MiB/s wr, 449 op/s 2024-09-13T13:00:24.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 114837 10.25 MB/sec execute 149 sec latency 44.466 ms 2024-09-13T13:00:25.484 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: Upgrade: Updating mon.smithi042 2024-09-13T13:00:25.484 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:25.485 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T13:00:25.485 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T13:00:25.485 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T13:00:25.485 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: Deploying daemon mon.smithi042 on smithi042 2024-09-13T13:00:25.485 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:25 smithi042 ceph-mon[23723]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 MiB/s wr, 255 op/s 2024-09-13T13:00:25.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 115285 10.24 MB/sec execute 150 sec latency 32.901 ms 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: Upgrade: Updating mon.smithi042 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: Deploying daemon mon.smithi042 on smithi042 2024-09-13T13:00:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:25 smithi046 ceph-mon[32268]: pgmap v25: 97 pgs: 97 active+clean; 334 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.4 MiB/s wr, 255 op/s 2024-09-13T13:00:26.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 115560 10.24 MB/sec execute 151 sec latency 119.925 ms 2024-09-13T13:00:26.745 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:26 smithi042 ceph-mon[23723]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T13:00:27.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:26 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T13:00:27.253 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:27 smithi042 systemd[1]: Stopping Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:00:27.520 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:27 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[23718]: 2024-09-13T13:00:27.322+0000 7f66a8232700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi042 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:00:27.520 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:27 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[23718]: 2024-09-13T13:00:27.322+0000 7f66a8232700 -1 mon.smithi042@0(leader) e2 *** Got Signal Terminated *** 2024-09-13T13:00:27.520 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:27 smithi042 podman[63806]: 2024-09-13 13:00:27.356340399 +0000 UTC m=+0.174751975 container died f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, GIT_CLEAN=True, io.buildah.version=1.19.8, version=8, release=754, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , architecture=x86_64, vendor=Red Hat, Inc.) 2024-09-13T13:00:27.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 115836 10.25 MB/sec execute 152 sec latency 84.047 ms 2024-09-13T13:00:28.551 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:28 smithi042 podman[63806]: 2024-09-13 13:00:28.31191633 +0000 UTC m=+1.130327905 container cleanup f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, ceph=True, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, RELEASE=HEAD, maintainer=Guillaume Abrioux , vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, architecture=x86_64, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-09-13T13:00:28.551 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:28 smithi042 bash[63806]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042 2024-09-13T13:00:28.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 116310 10.23 MB/sec execute 153 sec latency 54.935 ms 2024-09-13T13:00:28.782 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:28 smithi042 podman[63820]: 2024-09-13 13:00:28.697365987 +0000 UTC m=+1.332443886 container remove f35ac5b07f4e911358dc027adc5f66f00c9f0526c19d493e277a7380eb26b170 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, ceph=True, io.buildah.version=1.19.8, vendor=Red Hat, Inc., RELEASE=HEAD, name=centos-stream, vcs-type=git, release=754) 2024-09-13T13:00:29.073 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:28 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service: Deactivated successfully. 2024-09-13T13:00:29.073 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:28 smithi042 systemd[1]: Stopped Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:00:29.073 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:28 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service: Consumed 18.527s CPU time. 2024-09-13T13:00:29.575 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:29 smithi042 systemd[1]: Starting Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:00:29.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117041 10.30 MB/sec execute 154 sec latency 31.468 ms 2024-09-13T13:00:29.826 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:29 smithi042 podman[63953]: 2024-09-13 13:00:29.533331896 +0000 UTC m=+0.028560369 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:00:30.095 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:29 smithi042 podman[63953]: 2024-09-13 13:00:29.899723309 +0000 UTC m=+0.394951782 container create e5fd881ced8f144de80c1ee25733aadb129cba574d4a3c95bdf7747e28700e19 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 podman[63953]: 2024-09-13 13:00:30.296487344 +0000 UTC m=+0.791715826 container init e5fd881ced8f144de80c1ee25733aadb129cba574d4a3c95bdf7747e28700e19 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 podman[63953]: 2024-09-13 13:00:30.30809245 +0000 UTC m=+0.803320924 container start e5fd881ced8f144de80c1ee25733aadb129cba574d4a3c95bdf7747e28700e19 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev), process ceph-mon, pid 2 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: pidfile_write: ignore empty --pid-file 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: load: jerasure load: lrc 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: RocksDB version: 7.9.2 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Git sha 0 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Compile date 2024-09-13 10:06:52 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: DB SUMMARY 2024-09-13T13:00:30.348 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: DB Session ID: X7PXX184K9WL7M2IXE78 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: CURRENT file: CURRENT 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: MANIFEST file: MANIFEST-000015 size: 956 Bytes 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi042/store.db dir, Total Num: 1, files: 000030.sst 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi042/store.db: 000028.log size: 6108287 ; 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.error_if_exists: 0 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.create_if_missing: 0 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.paranoid_checks: 1 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T13:00:30.349 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.env: 0x55bd62441120 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.info_log: 0x55bd635eb8e0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.statistics: (nil) 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.use_fsync: 0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_log_file_size: 0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.allow_fallocate: 1 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T13:00:30.350 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.use_direct_reads: 0 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.db_log_dir: 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.wal_dir: 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T13:00:30.351 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.write_buffer_manager: 0x55bd635efb80 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.unordered_write: 0 2024-09-13T13:00:30.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T13:00:30.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T13:00:30.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T13:00:30.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T13:00:30.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.row_cache: None 2024-09-13T13:00:30.353 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.wal_filter: None 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.two_write_queues: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.wal_compression: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.atomic_flush: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T13:00:30.354 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.log_readahead_size: 0 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_background_jobs: 2 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_background_compactions: -1 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_subcompactions: 1 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T13:00:30.355 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_open_files: -1 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T13:00:30.356 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_background_flushes: -1 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Compression algorithms supported: 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kZSTD supported: 0 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kXpressCompression supported: 0 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kBZip2Compression supported: 0 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kLZ4Compression supported: 1 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kZlibCompression supported: 1 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: kSnappyCompression supported: 1 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T13:00:30.357 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi042/store.db/MANIFEST-000015 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.merge_operator: 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_filter: None 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bd635eb540) 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: cache_index_and_filter_blocks: 1 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T13:00:30.358 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: pin_top_level_index_and_filter: 1 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: index_type: 0 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: data_block_index_type: 0 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: index_shortening: 1 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: checksum: 4 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: no_block_cache: 0 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache: 0x55bd635df610 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache_name: BinnedLRUCache 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache_options: 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: capacity : 536870912 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: num_shard_bits : 4 2024-09-13T13:00:30.359 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: strict_capacity_limit : 0 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: high_pri_pool_ratio: 0.000 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_cache_compressed: (nil) 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: persistent_cache: (nil) 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_size: 4096 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_size_deviation: 10 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_restart_interval: 16 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: index_block_restart_interval: 1 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: metadata_block_size: 4096 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: partition_filters: 0 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: use_delta_encoding: 1 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: filter_policy: bloomfilter 2024-09-13T13:00:30.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: whole_key_filtering: 1 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: verify_compression: 0 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: read_amp_bytes_per_bit: 0 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: format_version: 5 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: enable_index_compression: 1 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: block_align: 0 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: max_auto_readahead_size: 262144 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: prepopulate_block_cache: 0 2024-09-13T13:00:30.362 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: initial_auto_readahead_size: 8192 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression: NoCompression 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.num_levels: 7 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T13:00:30.363 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T13:00:30.364 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T13:00:30.365 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T13:00:30.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T13:00:30.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T13:00:30.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T13:00:30.366 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T13:00:30.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T13:00:30.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T13:00:30.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T13:00:30.368 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T13:00:30.369 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.inplace_update_support: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.bloom_locality: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.max_successive_merges: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T13:00:30.370 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.ttl: 2592000 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.enable_blob_files: false 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.min_blob_size: 0 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T13:00:30.371 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi042/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 11636, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 068c3d00-9ff0-40eb-85f3-10586e31a93a 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726232430340713, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-13T13:00:30.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726232430361958, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 5496621, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11637, "largest_seqno": 14956, "table_properties": {"data_size": 5482791, "index_size": 9007, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 3781, "raw_key_size": 33533, "raw_average_key_size": 22, "raw_value_size": 5452282, "raw_average_value_size": 3634, "num_data_blocks": 403, "num_entries": 1500, "num_filter_entries": 1500, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726232430, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "068c3d00-9ff0-40eb-85f3-10586e31a93a", "db_session_id": "X7PXX184K9WL7M2IXE78", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726232430362063, "job": 1, "event": "recovery_finished"} 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi042/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bd6360ee00 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: DB pointer 0x55bd63710000 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T13:00:30.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** DB Stats ** 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: L0 1/0 5.24 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 585.9 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: L6 1/0 4.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T13:00:30.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Sum 2/0 10.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 585.9 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 585.9 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** Compaction Stats [default] ** 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 585.9 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Flush(GB): cumulative 0.005, interval 0.005 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T13:00:30.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Cumulative compaction: 0.01 GB write, 216.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Interval compaction: 0.01 GB write, 216.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Block cache BinnedLRUCache@0x55bd635df610#2 capacity: 512.00 MB usage: 13.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: starting mon.smithi042 rank 0 at public addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] at bind addrs [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi042 fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???) e2 preinit fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).mds e7 new map 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).mds e7 print_map 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: e7 2024-09-13T13:00:30.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: legacy client fscid: 1 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Filesystem 'foofs' (1) 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: fs_name foofs 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: epoch 5 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: created 2024-09-13T12:55:14.957607+0000 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: modified 2024-09-13T12:55:21.980505+0000 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: tableserver 0 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: root 0 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: session_timeout 60 2024-09-13T13:00:30.603 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: session_autoclose 300 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: max_file_size 1099511627776 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: max_xattr_size 65536 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: required_client_features {} 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: last_failure 0 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: last_failure_osd_epoch 0 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: max_mds 1 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: in 0 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: up {0=14588} 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: failed 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: damaged 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: stopped 2024-09-13T13:00:30.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: data_pools [3] 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: metadata_pool 2 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: inline_data disabled 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: balancer 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: bal_rank_mask -1 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: standby_count_wanted 1 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: qdb_cluster leader: 0 members: 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: [mds.foofs.smithi042.hljzvq{0:14588} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.42:6834/54759589,v1:172.21.15.42:6835/54759589] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Standby daemons: 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: 2024-09-13T13:00:30.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: [mds.foofs.smithi046.nqnolk{-1:24289} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).osd e72 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).osd e72 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 ceph-mon[63970]: mon.smithi042@-1(???).paxosservice(auth 1..29) refresh upgraded, format 0 -> 3 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 bash[63953]: e5fd881ced8f144de80c1ee25733aadb129cba574d4a3c95bdf7747e28700e19 2024-09-13T13:00:30.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:30 smithi042 systemd[1]: Started Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:00:30.614 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117476 10.31 MB/sec execute 155 sec latency 90.955 ms 2024-09-13T13:00:31.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117697 10.30 MB/sec execute 156 sec latency 96.304 ms 2024-09-13T13:00:32.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117829 10.28 MB/sec execute 157 sec latency 795.171 ms 2024-09-13T13:00:33.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117829 10.21 MB/sec execute 158 sec latency 1795.222 ms 2024-09-13T13:00:34.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117829 10.15 MB/sec execute 159 sec latency 2795.258 ms 2024-09-13T13:00:35.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117829 10.08 MB/sec execute 160 sec latency 3795.682 ms 2024-09-13T13:00:36.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 117842 10.02 MB/sec execute 161 sec latency 4771.366 ms 2024-09-13T13:00:36.842 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 390 op/s 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 376 op/s 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: pgmap v29: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 619 op/s 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 497 op/s 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: mon.smithi042 calling monitor election 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: mon.smithi042 is new leader, mons smithi042,smithi046 in quorum (ranks 0,1) 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: monmap epoch 2 2024-09-13T13:00:36.843 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: last_changed 2024-09-13T12:49:57.082008+0000 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: created 2024-09-13T12:44:55.874236+0000 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: min_mon_release 17 (quincy) 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: election_strategy: 1 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.smithi042 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: 1: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.smithi046 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: mgrmap e35: smithi042.jhsfyp(active, since 56s), standbys: smithi046.hcgddm 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: overall HEALTH_OK 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: from='mgr.24507 ' entity='' 2024-09-13T13:00:36.844 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:36 smithi042 ceph-mon[63970]: mgrmap e36: smithi042.jhsfyp(active, since 56s), standbys: smithi046.hcgddm 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi042"}]: dispatch 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "mon metadata", "id": "smithi046"}]: dispatch 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: from='mgr.24507 172.21.15.42:0/2189945235' entity='mgr.smithi042.jhsfyp' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: pgmap v27: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.2 MiB/s wr, 390 op/s 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: pgmap v28: 97 pgs: 97 active+clean; 335 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.9 MiB/s wr, 376 op/s 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: pgmap v29: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 9.2 KiB/s rd, 6.4 MiB/s wr, 619 op/s 2024-09-13T13:00:37.026 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: pgmap v30: 97 pgs: 97 active+clean; 337 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.2 MiB/s wr, 497 op/s 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: mon.smithi042 calling monitor election 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: mon.smithi042 is new leader, mons smithi042,smithi046 in quorum (ranks 0,1) 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: monmap epoch 2 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: last_changed 2024-09-13T12:49:57.082008+0000 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: created 2024-09-13T12:44:55.874236+0000 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: min_mon_release 17 (quincy) 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: election_strategy: 1 2024-09-13T13:00:37.027 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.smithi042 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: 1: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.smithi046 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: osdmap e72: 8 total, 8 up, 8 in 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: mgrmap e35: smithi042.jhsfyp(active, since 56s), standbys: smithi046.hcgddm 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: overall HEALTH_OK 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: from='mgr.24507 ' entity='' 2024-09-13T13:00:37.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:36 smithi046 ceph-mon[32268]: mgrmap e36: smithi042.jhsfyp(active, since 56s), standbys: smithi046.hcgddm 2024-09-13T13:00:37.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 118209 10.02 MB/sec execute 162 sec latency 37.713 ms 2024-09-13T13:00:38.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 118539 9.98 MB/sec execute 163 sec latency 40.768 ms 2024-09-13T13:00:38.646 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:38 smithi042 ceph-mon[63970]: mgrmap e37: smithi042.jhsfyp(active, since 57s), standbys: smithi046.hcgddm 2024-09-13T13:00:38.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:38 smithi046 ceph-mon[32268]: mgrmap e37: smithi042.jhsfyp(active, since 57s), standbys: smithi046.hcgddm 2024-09-13T13:00:39.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 118789 9.94 MB/sec execute 164 sec latency 74.769 ms 2024-09-13T13:00:40.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 119053 9.93 MB/sec execute 165 sec latency 42.937 ms 2024-09-13T13:00:41.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 119325 9.94 MB/sec execute 166 sec latency 60.362 ms 2024-09-13T13:00:42.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 119645 9.92 MB/sec execute 167 sec latency 81.475 ms 2024-09-13T13:00:43.264 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:43 smithi042 ceph-mon[63970]: Standby manager daemon smithi046.hcgddm restarted 2024-09-13T13:00:43.264 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:43 smithi042 ceph-mon[63970]: Standby manager daemon smithi046.hcgddm started 2024-09-13T13:00:43.264 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:43 smithi042 ceph-mon[63970]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T13:00:43.264 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:43 smithi042 ceph-mon[63970]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T13:00:43.264 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:43 smithi042 ceph-mon[63970]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T13:00:43.264 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:43 smithi042 ceph-mon[63970]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T13:00:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:43 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm restarted 2024-09-13T13:00:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:43 smithi046 ceph-mon[32268]: Standby manager daemon smithi046.hcgddm started 2024-09-13T13:00:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/crt"}]: dispatch 2024-09-13T13:00:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T13:00:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi046.hcgddm/key"}]: dispatch 2024-09-13T13:00:43.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:43 smithi046 ceph-mon[32268]: from='mgr.? 172.21.15.46:0/3891656056' entity='mgr.smithi046.hcgddm' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T13:00:43.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 119966 9.90 MB/sec execute 168 sec latency 89.621 ms 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: mgrmap e38: smithi042.jhsfyp(active, since 63s), standbys: smithi046.hcgddm 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: Active manager daemon smithi042.jhsfyp restarted 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: Activating manager daemon smithi042.jhsfyp 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: mgrmap e39: smithi042.jhsfyp(active, starting, since 0.00630603s), standbys: smithi046.hcgddm 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi042"} : dispatch 2024-09-13T13:00:44.277 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi046"} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T13:00:44.278 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"} : dispatch 2024-09-13T13:00:44.279 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"} : dispatch 2024-09-13T13:00:44.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 120299 9.89 MB/sec execute 169 sec latency 51.341 ms 2024-09-13T13:00:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: mgrmap e38: smithi042.jhsfyp(active, since 63s), standbys: smithi046.hcgddm 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: Active manager daemon smithi042.jhsfyp restarted 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: Activating manager daemon smithi042.jhsfyp 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: mgrmap e39: smithi042.jhsfyp(active, starting, since 0.00630603s), standbys: smithi046.hcgddm 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi042"} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi046"} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr metadata", "who": "smithi042.jhsfyp", "id": "smithi042.jhsfyp"} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr metadata", "who": "smithi046.hcgddm", "id": "smithi046.hcgddm"} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T13:00:44.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata"} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata"} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata"} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: Manager daemon smithi042.jhsfyp is now available 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-13T13:00:44.619 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:00:44.620 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/mirror_snapshot_schedule"} : dispatch 2024-09-13T13:00:44.620 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:44 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi042.jhsfyp/trash_purge_schedule"} : dispatch 2024-09-13T13:00:45.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:45 smithi042 ceph-mon[63970]: mgrmap e40: smithi042.jhsfyp(active, since 1.01509s), standbys: smithi046.hcgddm 2024-09-13T13:00:45.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:45 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T13:00:45.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:45 smithi042 ceph-mon[63970]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:45.615 INFO:teuthology.orchestra.run.smithi042.stdout: 5 120619 9.90 MB/sec execute 170 sec latency 56.988 ms 2024-09-13T13:00:45.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:45 smithi046 ceph-mon[32268]: mgrmap e40: smithi042.jhsfyp(active, since 1.01509s), standbys: smithi046.hcgddm 2024-09-13T13:00:45.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:45 smithi046 ceph-mon[32268]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-13T13:00:45.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:45 smithi046 ceph-mon[32268]: pgmap v3: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:46.269 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:46 smithi046 ceph-mon[32268]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:46.269 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:46 smithi046 ceph-mon[32268]: mgrmap e41: smithi042.jhsfyp(active, since 2s), standbys: smithi046.hcgddm 2024-09-13T13:00:46.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:46 smithi042 ceph-mon[63970]: pgmap v4: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:46.352 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:46 smithi042 ceph-mon[63970]: mgrmap e41: smithi042.jhsfyp(active, since 2s), standbys: smithi046.hcgddm 2024-09-13T13:00:46.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 121039 9.92 MB/sec execute 171 sec latency 53.027 ms 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:47 smithi046 ceph-mon[32268]: [13/Sep/2024:13:00:46] ENGINE Bus STARTING 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:47 smithi046 ceph-mon[32268]: [13/Sep/2024:13:00:46] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:47 smithi046 ceph-mon[32268]: [13/Sep/2024:13:00:46] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:47 smithi046 ceph-mon[32268]: [13/Sep/2024:13:00:46] ENGINE Bus STARTED 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:47 smithi046 ceph-mon[32268]: [13/Sep/2024:13:00:46] ENGINE Error in HTTPServer.serve 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Traceback (most recent call last): 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T13:00:47.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._connections.run(self.expiration_interval) 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._run(expiration_interval) 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: s = self.context.wrap_socket( 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: return self.sslsocket_class._create( 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self.do_handshake() 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T13:00:47.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: self._sslobj.do_handshake() 2024-09-13T13:00:47.369 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T13:00:47.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:47 smithi042 ceph-mon[63970]: [13/Sep/2024:13:00:46] ENGINE Bus STARTING 2024-09-13T13:00:47.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:47 smithi042 ceph-mon[63970]: [13/Sep/2024:13:00:46] ENGINE Serving on http://172.21.15.42:8765 2024-09-13T13:00:47.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:47 smithi042 ceph-mon[63970]: [13/Sep/2024:13:00:46] ENGINE Serving on https://172.21.15.42:7150 2024-09-13T13:00:47.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:47 smithi042 ceph-mon[63970]: [13/Sep/2024:13:00:46] ENGINE Bus STARTED 2024-09-13T13:00:47.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:47 smithi042 ceph-mon[63970]: [13/Sep/2024:13:00:46] ENGINE Error in HTTPServer.serve 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: Traceback (most recent call last): 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._connections.run(self.expiration_interval) 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._run(expiration_interval) 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: s = self.context.wrap_socket( 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T13:00:47.385 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: return self.sslsocket_class._create( 2024-09-13T13:00:47.386 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T13:00:47.386 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self.do_handshake() 2024-09-13T13:00:47.386 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T13:00:47.386 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: self._sslobj.do_handshake() 2024-09-13T13:00:47.386 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T13:00:47.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 121373 9.92 MB/sec execute 172 sec latency 42.258 ms 2024-09-13T13:00:48.392 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:48 smithi042 ceph-mon[63970]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:48.392 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:48 smithi042 ceph-mon[63970]: mgrmap e42: smithi042.jhsfyp(active, since 4s), standbys: smithi046.hcgddm 2024-09-13T13:00:48.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 121620 9.91 MB/sec execute 173 sec latency 61.609 ms 2024-09-13T13:00:48.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:48 smithi046 ceph-mon[32268]: pgmap v5: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:48.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:48 smithi046 ceph-mon[32268]: mgrmap e42: smithi042.jhsfyp(active, since 4s), standbys: smithi046.hcgddm 2024-09-13T13:00:49.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 121999 9.89 MB/sec execute 174 sec latency 58.576 ms 2024-09-13T13:00:49.678 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:49 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.678 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:49 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.678 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:49 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.678 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:49 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.929 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:49 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.929 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:49 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.929 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:49 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:49.930 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:49 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:50.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 122290 9.86 MB/sec execute 175 sec latency 113.309 ms 2024-09-13T13:00:50.706 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:50 smithi042 ceph-mon[63970]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:50.951 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:50 smithi046 ceph-mon[32268]: pgmap v6: 97 pgs: 97 active+clean; 337 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-13T13:00:51.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 122585 9.86 MB/sec execute 176 sec latency 51.588 ms 2024-09-13T13:00:52.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 122910 9.87 MB/sec execute 177 sec latency 64.805 ms 2024-09-13T13:00:52.766 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:52 smithi046 ceph-mon[32268]: pgmap v7: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.8 MiB/s wr, 355 op/s 2024-09-13T13:00:52.767 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:00:52.882 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:52 smithi042 ceph-mon[63970]: pgmap v7: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 30 KiB/s rd, 3.8 MiB/s wr, 355 op/s 2024-09-13T13:00:53.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 123287 9.85 MB/sec execute 178 sec latency 53.020 ms 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (2m) 1s ago 14m 21.9M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (14m) 1s ago 14m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (11m) 5s ago 11m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (2m) 1s ago 11m 43.8M - dad864ee21e9 7144014c89b8 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (5m) 1s ago 5m 40.5M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (5m) 5s ago 5m 15.3M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (90s) 1s ago 15m 488M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (41s) 5s ago 11m 451M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:00:53.862 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (24s) 1s ago 15m 34.4M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (10m) 5s ago 10m 56.1M 2048M 17.2.0 e1d6a67b021e 0e99d3e715a3 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (5m) 1s ago 5m 262M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (5m) 5s ago 5m 87.6M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (2m) 1s ago 11m 9773k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (99s) 5s ago 10m 11.3M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (10m) 1s ago 10m 461M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (9m) 1s ago 9m 398M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (9m) 1s ago 9m 264M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (8m) 1s ago 8m 413M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (8m) 5s ago 8m 425M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (7m) 5s ago 7m 459M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (7m) 5s ago 7m 402M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:00:53.863 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (7m) 5s ago 7m 551M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:00:53.864 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (48s) 1s ago 10m 44.1M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:00:53.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:53.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:53.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:53.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: pgmap v8: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T13:00:53.895 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:53 smithi042 ceph-mon[63970]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T13:00:54.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='client.34128 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: pgmap v8: 97 pgs: 97 active+clean; 342 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 2.9 MiB/s wr, 276 op/s 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='client.34132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.conf 2024-09-13T13:00:54.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:53 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.conf 2024-09-13T13:00:54.474 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:00:54.474 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:00:54.474 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T13:00:54.474 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 11, 2024-09-13T13:00:54.475 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 3 2024-09-13T13:00:54.476 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:00:54.476 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:00:54.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 123625 9.83 MB/sec execute 179 sec latency 96.458 ms 2024-09-13T13:00:54.975 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:00:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:00:54.975 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "3/23 daemons upgraded", 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:00:54.976 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:00:55.430 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T13:00:55.430 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T13:00:55.430 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/3091094814' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:00:55.431 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:00:55.432 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T13:00:55.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 123825 9.81 MB/sec execute 180 sec latency 79.219 ms 2024-09-13T13:00:55.639 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='client.34136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: Updating smithi042:/etc/ceph/ceph.client.admin.keyring 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: Updating smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: Updating smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.684 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:55.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/3091094814' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:00:55.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:00:55.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:00:55.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:55 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "quorum_status"} : dispatch 2024-09-13T13:00:56.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 124216 9.85 MB/sec execute 181 sec latency 67.886 ms 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1164212319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T13:00:56.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:00:56.791 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: from='client.34144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:00:56.800 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: pgmap v9: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 23 KiB/s rd, 4.5 MiB/s wr, 448 op/s 2024-09-13T13:00:56.800 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: from='client.? 172.21.15.42:0/1164212319' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-09-13T13:00:56.800 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:00:56.800 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T13:00:56.800 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T13:00:56.800 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:56 smithi046 ceph-mon[32268]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:00:57.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 124569 9.83 MB/sec execute 182 sec latency 40.766 ms 2024-09-13T13:00:57.725 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:57 smithi042 ceph-mon[63970]: Upgrade: Updating mon.smithi046 2024-09-13T13:00:57.725 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:57 smithi042 ceph-mon[63970]: Deploying daemon mon.smithi046 on smithi046 2024-09-13T13:00:57.830 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:57 smithi046 ceph-mon[32268]: Upgrade: Updating mon.smithi046 2024-09-13T13:00:57.831 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:57 smithi046 ceph-mon[32268]: Deploying daemon mon.smithi046 on smithi046 2024-09-13T13:00:58.490 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:00:58 smithi042 ceph-mon[63970]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-13T13:00:58.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 124880 9.83 MB/sec execute 183 sec latency 41.183 ms 2024-09-13T13:00:58.738 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:58 smithi046 systemd[1]: Stopping Ceph mon.smithi046 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:00:58.738 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:58 smithi046 ceph-mon[32268]: pgmap v10: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 21 KiB/s rd, 4.1 MiB/s wr, 410 op/s 2024-09-13T13:00:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:58 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046[32250]: 2024-09-13T13:00:58.736+0000 7f751aa9a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi046 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:00:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:58 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046[32250]: 2024-09-13T13:00:58.736+0000 7f751aa9a700 -1 mon.smithi046@1(peon) e2 *** Got Signal Terminated *** 2024-09-13T13:00:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:58 smithi046 podman[55817]: 2024-09-13 13:00:58.961401413 +0000 UTC m=+0.333667217 container died 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, build-date=2022-05-03T08:36:31.336870, name=centos-stream, release=754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, version=8, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, ceph=True, distribution-scope=public) 2024-09-13T13:00:59.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 125079 9.82 MB/sec execute 184 sec latency 54.924 ms 2024-09-13T13:00:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:59 smithi046 podman[55817]: 2024-09-13 13:00:59.327705168 +0000 UTC m=+0.699970980 container cleanup 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , release=754, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, vcs-type=git, RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.openshift.tags=base centos centos-stream) 2024-09-13T13:00:59.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:59 smithi046 bash[55817]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046 2024-09-13T13:01:00.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:59 smithi046 podman[55831]: 2024-09-13 13:00:59.621415885 +0000 UTC m=+0.660309350 container remove 0e99d3e715a331e2b3e2766d29d2e4bd48d3fc77b35429ed43e1abf739d27ae1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., io.buildah.version=1.19.8, release=754, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, distribution-scope=public, ceph=True, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, version=8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870) 2024-09-13T13:01:00.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:59 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi046.service: Deactivated successfully. 2024-09-13T13:01:00.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:59 smithi046 systemd[1]: Stopped Ceph mon.smithi046 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:01:00.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:00:59 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi046.service: Consumed 7.642s CPU time. 2024-09-13T13:01:00.339 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 systemd[1]: Starting Ceph mon.smithi046 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:01:00.339 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 podman[55953]: 2024-09-13 13:01:00.220749179 +0000 UTC m=+0.020997871 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:01:00.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 125339 9.81 MB/sec execute 185 sec latency 56.117 ms 2024-09-13T13:01:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 podman[55953]: 2024-09-13 13:01:00.339200831 +0000 UTC m=+0.139449510 container create ff1c51b08a869e0cbd4175e6613debe3c98dcec086e8854a8c862a51a3c5b19e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:01:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 podman[55953]: 2024-09-13 13:01:00.583747708 +0000 UTC m=+0.383996388 container init ff1c51b08a869e0cbd4175e6613debe3c98dcec086e8854a8c862a51a3c5b19e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T13:01:00.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 podman[55953]: 2024-09-13 13:01:00.588550087 +0000 UTC m=+0.388798773 container start ff1c51b08a869e0cbd4175e6613debe3c98dcec086e8854a8c862a51a3c5b19e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: set uid:gid to 167:167 (ceph:ceph) 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev), process ceph-mon, pid 2 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: pidfile_write: ignore empty --pid-file 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: load: jerasure load: lrc 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: RocksDB version: 7.9.2 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Git sha 0 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Compile date 2024-09-13 10:06:52 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: DB SUMMARY 2024-09-13T13:01:01.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: DB Session ID: NOCZJHNFK3O0CFJUNGMY 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: CURRENT file: CURRENT 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: IDENTITY file: IDENTITY 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: MANIFEST file: MANIFEST-000009 size: 881 Bytes 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi046/store.db dir, Total Num: 1, files: 000024.sst 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi046/store.db: 000022.log size: 12642284 ; 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.error_if_exists: 0 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.create_if_missing: 0 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.paranoid_checks: 1 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.env: 0x55b4b0b9d120 2024-09-13T13:01:01.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.fs: PosixFileSystem 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.info_log: 0x55b4b25978e0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_file_opening_threads: 16 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.statistics: (nil) 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.use_fsync: 0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_log_file_size: 0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.keep_log_file_num: 1000 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.recycle_log_file_num: 0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.allow_fallocate: 1 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.allow_mmap_reads: 0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.allow_mmap_writes: 0 2024-09-13T13:01:01.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.use_direct_reads: 0 2024-09-13T13:01:01.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T13:01:01.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.create_missing_column_families: 0 2024-09-13T13:01:01.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.db_log_dir: 2024-09-13T13:01:01.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.wal_dir: 2024-09-13T13:01:01.121 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T13:01:01.122 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T13:01:01.122 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T13:01:01.122 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.advise_random_on_open: 1 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.db_write_buffer_size: 0 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.write_buffer_manager: 0x55b4b259bb80 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.rate_limiter: (nil) 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.wal_recovery_mode: 2 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.enable_thread_tracking: 0 2024-09-13T13:01:01.123 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.enable_pipelined_write: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.unordered_write: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.row_cache: None 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.wal_filter: None 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.allow_ingest_behind: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.two_write_queues: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.manual_wal_flush: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.wal_compression: 0 2024-09-13T13:01:01.124 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.atomic_flush: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.log_readahead_size: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.best_efforts_recovery: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.allow_data_in_errors: 0 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.db_host_id: __hostname__ 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_background_jobs: 2 2024-09-13T13:01:01.125 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_background_compactions: -1 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_subcompactions: 1 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_total_wal_size: 0 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_open_files: -1 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bytes_per_sync: 0 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T13:01:01.126 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_readahead_size: 0 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_background_flushes: -1 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Compression algorithms supported: 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kZSTD supported: 0 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kXpressCompression supported: 0 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kBZip2Compression supported: 0 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kLZ4Compression supported: 1 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kZlibCompression supported: 1 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kLZ4HCCompression supported: 1 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: kSnappyCompression supported: 1 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi046/store.db/MANIFEST-000009 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T13:01:01.127 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.merge_operator: 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_filter: None 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_filter_factory: None 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.sst_partitioner_factory: None 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55b4b2597540) 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: cache_index_and_filter_blocks: 1 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: pin_top_level_index_and_filter: 1 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: index_type: 0 2024-09-13T13:01:01.128 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: data_block_index_type: 0 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: index_shortening: 1 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: checksum: 4 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: no_block_cache: 0 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache: 0x55b4b258b610 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache_name: BinnedLRUCache 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache_options: 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: capacity : 536870912 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: num_shard_bits : 4 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: strict_capacity_limit : 0 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: high_pri_pool_ratio: 0.000 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_cache_compressed: (nil) 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: persistent_cache: (nil) 2024-09-13T13:01:01.129 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_size: 4096 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_size_deviation: 10 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_restart_interval: 16 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: index_block_restart_interval: 1 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: metadata_block_size: 4096 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: partition_filters: 0 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: use_delta_encoding: 1 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: filter_policy: bloomfilter 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: whole_key_filtering: 1 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: verify_compression: 0 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: read_amp_bytes_per_bit: 0 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: format_version: 5 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: enable_index_compression: 1 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: block_align: 0 2024-09-13T13:01:01.130 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: max_auto_readahead_size: 262144 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: prepopulate_block_cache: 0 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: initial_auto_readahead_size: 8192 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: num_file_reads_for_auto_readahead: 2 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.write_buffer_size: 33554432 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_write_buffer_number: 2 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression: NoCompression 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression: Disabled 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.prefix_extractor: nullptr 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.num_levels: 7 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T13:01:01.131 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.level: 32767 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.strategy: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T13:01:01.132 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.enabled: false 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.target_file_size_base: 67108864 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T13:01:01.133 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.arena_block_size: 1048576 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.disable_auto_compactions: 0 2024-09-13T13:01:01.134 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.inplace_update_support: 0 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T13:01:01.135 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.bloom_locality: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.max_successive_merges: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.paranoid_file_checks: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.force_consistency_checks: 1 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.report_bg_io_stats: 0 2024-09-13T13:01:01.136 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.ttl: 2592000 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.enable_blob_files: false 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.min_blob_size: 0 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.blob_file_size: 268435456 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.blob_file_starting_level: 0 2024-09-13T13:01:01.137 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi046/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 11126, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 41aa36f7-db67-4ed3-b936-1be50916fad8 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726232460625208, "job": 1, "event": "recovery_started", "wal_files": [22]} 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726232460657318, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 7601331, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 11131, "largest_seqno": 15336, "table_properties": {"data_size": 7585275, "index_size": 10657, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4357, "raw_key_size": 38910, "raw_average_key_size": 22, "raw_value_size": 7549957, "raw_average_value_size": 4364, "num_data_blocks": 476, "num_entries": 1730, "num_filter_entries": 1730, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726232460, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "41aa36f7-db67-4ed3-b936-1be50916fad8", "db_session_id": "NOCZJHNFK3O0CFJUNGMY", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726232460657480, "job": 1, "event": "recovery_finished"} 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2024-09-13T13:01:01.138 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-smithi046/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55b4b25bae00 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: DB pointer 0x55b4b26bc000 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** DB Stats ** 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T13:01:01.139 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** Compaction Stats [default] ** 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: L0 1/0 7.25 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 606.5 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: L6 1/0 4.90 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Sum 2/0 12.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 606.5 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 606.5 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** Compaction Stats [default] ** 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 606.5 0.01 0.00 1 0.012 0 0 0.0 0.0 2024-09-13T13:01:01.140 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Cumulative compaction: 0.01 GB write, 195.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Interval compaction: 0.01 GB write, 195.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T13:01:01.141 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Block cache BinnedLRUCache@0x55b4b258b610#2 capacity: 512.00 MB usage: 16.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: starting mon.smithi046 rank 1 at public addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] at bind addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi046 fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???) e2 preinit fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).mds e7 new map 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).mds e7 print_map 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: e7 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: legacy client fscid: 1 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.142 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Filesystem 'foofs' (1) 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: fs_name foofs 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: epoch 5 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: created 2024-09-13T12:55:14.957607+0000 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: modified 2024-09-13T12:55:21.980505+0000 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: tableserver 0 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: root 0 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: session_timeout 60 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: session_autoclose 300 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: max_file_size 1099511627776 2024-09-13T13:01:01.143 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: max_xattr_size 65536 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: required_client_features {} 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: last_failure 0 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: last_failure_osd_epoch 0 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: max_mds 1 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: in 0 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: up {0=14588} 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: failed 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: damaged 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: stopped 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: data_pools [3] 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: metadata_pool 2 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: inline_data disabled 2024-09-13T13:01:01.144 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: balancer 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: bal_rank_mask -1 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: standby_count_wanted 1 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: qdb_cluster leader: 0 members: 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: [mds.foofs.smithi042.hljzvq{0:14588} state up:active seq 2 join_fscid=1 addr [v2:172.21.15.42:6834/54759589,v1:172.21.15.42:6835/54759589] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: Standby daemons: 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout: [mds.foofs.smithi046.nqnolk{-1:24289} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).osd e73 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).osd e73 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T13:01:01.145 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 ceph-mon[55972]: mon.smithi046@-1(???).paxosservice(auth 1..30) refresh upgraded, format 0 -> 3 2024-09-13T13:01:01.146 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 bash[55953]: ff1c51b08a869e0cbd4175e6613debe3c98dcec086e8854a8c862a51a3c5b19e 2024-09-13T13:01:01.146 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:00 smithi046 systemd[1]: Started Ceph mon.smithi046 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:01:01.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 125677 9.77 MB/sec execute 186 sec latency 97.137 ms 2024-09-13T13:01:02.056 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: mgrmap e42: smithi042.jhsfyp(active, since 17s), standbys: smithi046.hcgddm 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: overall HEALTH_OK 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: mon.smithi042 calling monitor election 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: mon.smithi046 calling monitor election 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi042"} : dispatch 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi046"} : dispatch 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: mon.smithi042 is new leader, mons smithi042,smithi046 in quorum (ranks 0,1) 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: monmap epoch 3 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: last_changed 2024-09-13T13:01:00.958419+0000 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: created 2024-09-13T12:44:55.874236+0000 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: min_mon_release 19 (squid) 2024-09-13T13:01:02.057 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: election_strategy: 1 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.smithi042 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: 1: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.smithi046 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: mgrmap e42: smithi042.jhsfyp(active, since 17s), standbys: smithi046.hcgddm 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: overall HEALTH_OK 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:01:02.058 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:01 smithi042 ceph-mon[63970]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: mgrmap e42: smithi042.jhsfyp(active, since 17s), standbys: smithi046.hcgddm 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: overall HEALTH_OK 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: mon.smithi042 calling monitor election 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: mon.smithi046 calling monitor election 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi042"} : dispatch 2024-09-13T13:01:02.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mon metadata", "id": "smithi046"} : dispatch 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: mon.smithi042 is new leader, mons smithi042,smithi046 in quorum (ranks 0,1) 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: monmap epoch 3 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: last_changed 2024-09-13T13:01:00.958419+0000 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: created 2024-09-13T12:44:55.874236+0000 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: min_mon_release 19 (squid) 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: election_strategy: 1 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: 0: [v2:172.21.15.42:3300/0,v1:172.21.15.42:6789/0] mon.smithi042 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: 1: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.smithi046 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: osdmap e73: 8 total, 8 up, 8 in 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: mgrmap e42: smithi042.jhsfyp(active, since 17s), standbys: smithi046.hcgddm 2024-09-13T13:01:02.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: overall HEALTH_OK 2024-09-13T13:01:02.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:02.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:02.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:01:02.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:01 smithi046 ceph-mon[55972]: pgmap v12: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 24 KiB/s rd, 6.2 MiB/s wr, 601 op/s 2024-09-13T13:01:02.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 125943 9.76 MB/sec execute 187 sec latency 49.112 ms 2024-09-13T13:01:03.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 126275 9.77 MB/sec execute 188 sec latency 38.980 ms 2024-09-13T13:01:04.372 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:04 smithi042 ceph-mon[63970]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-13T13:01:04.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 126672 9.77 MB/sec execute 189 sec latency 60.189 ms 2024-09-13T13:01:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:04 smithi046 ceph-mon[55972]: pgmap v13: 97 pgs: 97 active+clean; 340 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s 2024-09-13T13:01:05.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 127070 9.76 MB/sec execute 190 sec latency 51.555 ms 2024-09-13T13:01:06.172 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:05 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.172 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:05 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.172 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:05 smithi042 ceph-mon[63970]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.8 MiB/s wr, 581 op/s 2024-09-13T13:01:06.172 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:05 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.172 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:05 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.191 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:05 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.191 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:05 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.191 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:05 smithi046 ceph-mon[55972]: pgmap v14: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.8 MiB/s wr, 581 op/s 2024-09-13T13:01:06.191 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:05 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.191 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:05 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:06.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 127307 9.73 MB/sec execute 191 sec latency 62.737 ms 2024-09-13T13:01:07.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 127586 9.74 MB/sec execute 192 sec latency 54.692 ms 2024-09-13T13:01:08.482 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:08 smithi042 ceph-mon[63970]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-13T13:01:08.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 127916 9.74 MB/sec execute 193 sec latency 80.871 ms 2024-09-13T13:01:08.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:08 smithi046 ceph-mon[55972]: pgmap v15: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-13T13:01:09.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 128175 9.72 MB/sec execute 194 sec latency 93.306 ms 2024-09-13T13:01:10.579 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-13T13:01:10.579 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.579 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.579 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:10.579 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: Reconfiguring mon.smithi042 (monmap changed)... 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:10.580 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:10 smithi042 ceph-mon[63970]: Reconfiguring daemon mon.smithi042 on smithi042 2024-09-13T13:01:10.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 128480 9.73 MB/sec execute 195 sec latency 80.742 ms 2024-09-13T13:01:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: pgmap v16: 97 pgs: 97 active+clean; 340 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 377 op/s 2024-09-13T13:01:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:01:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: Reconfiguring mon.smithi042 (monmap changed)... 2024-09-13T13:01:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T13:01:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T13:01:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:10.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:10 smithi046 ceph-mon[55972]: Reconfiguring daemon mon.smithi042 on smithi042 2024-09-13T13:01:11.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 128751 9.74 MB/sec execute 196 sec latency 55.285 ms 2024-09-13T13:01:12.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 129002 9.71 MB/sec execute 197 sec latency 48.982 ms 2024-09-13T13:01:12.657 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:12 smithi042 ceph-mon[63970]: pgmap v17: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-09-13T13:01:12.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:12.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:12.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T13:01:12.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T13:01:12.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:12 smithi046 ceph-mon[55972]: pgmap v17: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.8 MiB/s wr, 562 op/s 2024-09-13T13:01:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi042.jhsfyp", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T13:01:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T13:01:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:13.616 INFO:teuthology.orchestra.run.smithi042.stdout: 5 129350 9.69 MB/sec execute 198 sec latency 71.745 ms 2024-09-13T13:01:13.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:13 smithi042 ceph-mon[63970]: Reconfiguring mgr.smithi042.jhsfyp (monmap changed)... 2024-09-13T13:01:13.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:13 smithi042 ceph-mon[63970]: Reconfiguring daemon mgr.smithi042.jhsfyp on smithi042 2024-09-13T13:01:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:13 smithi046 ceph-mon[55972]: Reconfiguring mgr.smithi042.jhsfyp (monmap changed)... 2024-09-13T13:01:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:13 smithi046 ceph-mon[55972]: Reconfiguring daemon mgr.smithi042.jhsfyp on smithi042 2024-09-13T13:01:14.617 INFO:teuthology.orchestra.run.smithi042.stdout: 5 129563 9.67 MB/sec execute 199 sec latency 58.283 ms 2024-09-13T13:01:14.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:14 smithi042 ceph-mon[63970]: pgmap v18: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-09-13T13:01:14.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:14.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:14.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:14.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:14.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:14 smithi046 ceph-mon[55972]: pgmap v18: 97 pgs: 97 active+clean; 349 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.7 MiB/s wr, 371 op/s 2024-09-13T13:01:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:15.617 INFO:teuthology.orchestra.run.smithi042.stdout: 5 129839 9.67 MB/sec execute 200 sec latency 52.027 ms 2024-09-13T13:01:15.720 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:15 smithi042 ceph-mon[63970]: Reconfiguring crash.smithi042 (monmap changed)... 2024-09-13T13:01:15.720 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:15 smithi042 ceph-mon[63970]: Reconfiguring daemon crash.smithi042 on smithi042 2024-09-13T13:01:15.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:15 smithi046 ceph-mon[55972]: Reconfiguring crash.smithi042 (monmap changed)... 2024-09-13T13:01:15.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:15 smithi046 ceph-mon[55972]: Reconfiguring daemon crash.smithi042 on smithi042 2024-09-13T13:01:16.617 INFO:teuthology.orchestra.run.smithi042.stdout: 5 130183 9.66 MB/sec execute 201 sec latency 40.764 ms 2024-09-13T13:01:16.728 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:16 smithi042 ceph-mon[63970]: pgmap v19: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-13T13:01:16.728 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:16 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:16.728 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:16 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:16.728 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:16 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T13:01:16.728 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:16 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:16 smithi046 ceph-mon[55972]: pgmap v19: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 548 op/s 2024-09-13T13:01:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:16 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:16 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:16 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T13:01:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:16 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:17.617 INFO:teuthology.orchestra.run.smithi042.stdout: 5 130555 9.64 MB/sec execute 202 sec latency 85.529 ms 2024-09-13T13:01:17.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:17 smithi042 ceph-mon[63970]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T13:01:17.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:17 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.0 on smithi042 2024-09-13T13:01:17.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:17 smithi046 ceph-mon[55972]: Reconfiguring osd.0 (monmap changed)... 2024-09-13T13:01:17.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:17 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.0 on smithi042 2024-09-13T13:01:18.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:18 smithi042 ceph-mon[63970]: pgmap v20: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T13:01:18.617 INFO:teuthology.orchestra.run.smithi042.stdout: 5 130808 9.62 MB/sec execute 203 sec latency 53.811 ms 2024-09-13T13:01:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:18 smithi046 ceph-mon[55972]: pgmap v20: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T13:01:19.617 INFO:teuthology.orchestra.run.smithi042.stdout: 5 130990 9.59 MB/sec execute 204 sec latency 49.329 ms 2024-09-13T13:01:19.809 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:19.809 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:19.809 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T13:01:19.809 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:20.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:20.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:20.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T13:01:20.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:20.619 INFO:teuthology.orchestra.run.smithi042.stdout: 5 131284 9.62 MB/sec execute 205 sec latency 70.759 ms 2024-09-13T13:01:20.829 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:20 smithi042 ceph-mon[63970]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T13:01:20.829 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:20 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.1 on smithi042 2024-09-13T13:01:20.829 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:20 smithi042 ceph-mon[63970]: pgmap v21: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T13:01:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:20 smithi046 ceph-mon[55972]: Reconfiguring osd.1 (monmap changed)... 2024-09-13T13:01:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:20 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.1 on smithi042 2024-09-13T13:01:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:20 smithi046 ceph-mon[55972]: pgmap v21: 97 pgs: 97 active+clean; 345 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.6 MiB/s wr, 361 op/s 2024-09-13T13:01:21.619 INFO:teuthology.orchestra.run.smithi042.stdout: 5 131756 9.62 MB/sec execute 206 sec latency 58.785 ms 2024-09-13T13:01:22.100 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:22.100 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:22.100 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T13:01:22.101 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T13:01:22.101 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:22.101 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.2 on smithi042 2024-09-13T13:01:22.101 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:21 smithi042 ceph-mon[63970]: pgmap v22: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 547 op/s 2024-09-13T13:01:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: Reconfiguring osd.2 (monmap changed)... 2024-09-13T13:01:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T13:01:22.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:22.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.2 on smithi042 2024-09-13T13:01:22.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:21 smithi046 ceph-mon[55972]: pgmap v22: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.4 MiB/s wr, 547 op/s 2024-09-13T13:01:22.619 INFO:teuthology.orchestra.run.smithi042.stdout: 5 132188 9.65 MB/sec execute 207 sec latency 37.313 ms 2024-09-13T13:01:23.619 INFO:teuthology.orchestra.run.smithi042.stdout: 5 132661 9.66 MB/sec execute 208 sec latency 39.908 ms 2024-09-13T13:01:24.619 INFO:teuthology.orchestra.run.smithi042.stdout: 5 133062 9.65 MB/sec execute 209 sec latency 37.179 ms 2024-09-13T13:01:24.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: pgmap v23: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-09-13T13:01:24.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:24.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:24.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T13:01:24.645 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T13:01:24.645 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:24.645 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:24 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.3 on smithi042 2024-09-13T13:01:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: pgmap v23: 97 pgs: 97 active+clean; 338 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 3.5 MiB/s wr, 362 op/s 2024-09-13T13:01:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: Reconfiguring osd.3 (monmap changed)... 2024-09-13T13:01:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T13:01:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:24.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:24 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.3 on smithi042 2024-09-13T13:01:25.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 133402 9.66 MB/sec execute 210 sec latency 49.648 ms 2024-09-13T13:01:26.296 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:01:26.611 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:26 smithi042 ceph-mon[63970]: pgmap v24: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.0 MiB/s wr, 591 op/s 2024-09-13T13:01:26.611 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:26 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:26.611 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:26 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:26.612 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:26 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:01:26.612 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:26 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:26.619 INFO:teuthology.orchestra.run.smithi042.stdout: 5 133774 9.66 MB/sec execute 211 sec latency 56.023 ms 2024-09-13T13:01:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:26 smithi046 ceph-mon[55972]: pgmap v24: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 6.0 MiB/s wr, 591 op/s 2024-09-13T13:01:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:26 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:26 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:26 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:01:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:26 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:27.545 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:01:27.545 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (2m) 34s ago 15m 21.9M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:01:27.545 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (15m) 34s ago 15m 6395k - 17.2.0 e1d6a67b021e c7de321065e6 2024-09-13T13:01:27.545 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (11m) 22s ago 11m 6409k - 17.2.0 e1d6a67b021e 4e8589041714 2024-09-13T13:01:27.545 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (2m) 34s ago 12m 43.8M - dad864ee21e9 7144014c89b8 2024-09-13T13:01:27.545 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (6m) 34s ago 6m 40.5M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (6m) 22s ago 6m 15.4M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (2m) 34s ago 16m 488M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (75s) 22s ago 11m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (57s) 34s ago 16m 34.4M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (27s) 22s ago 11m 30.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (5m) 34s ago 5m 262M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (5m) 22s ago 5m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (2m) 34s ago 12m 9773k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (2m) 22s ago 11m 12.2M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (10m) 34s ago 10m 461M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (10m) 34s ago 10m 398M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (9m) 34s ago 9m 264M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (9m) 34s ago 9m 413M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (8m) 22s ago 8m 453M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:01:27.546 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (8m) 22s ago 8m 488M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:01:27.547 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (8m) 22s ago 8m 412M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:01:27.547 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (7m) 22s ago 7m 574M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:01:27.547 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (81s) 34s ago 11m 44.1M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:01:27.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 134213 9.65 MB/sec execute 212 sec latency 42.248 ms 2024-09-13T13:01:27.624 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:27 smithi042 ceph-mon[63970]: Reconfiguring mds.foofs.smithi042.hljzvq (monmap changed)... 2024-09-13T13:01:27.624 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:27 smithi042 ceph-mon[63970]: Reconfiguring daemon mds.foofs.smithi042.hljzvq on smithi042 2024-09-13T13:01:27.624 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:27 smithi042 ceph-mon[63970]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:27.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:27 smithi046 ceph-mon[55972]: Reconfiguring mds.foofs.smithi042.hljzvq (monmap changed)... 2024-09-13T13:01:27.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:27 smithi046 ceph-mon[55972]: Reconfiguring daemon mds.foofs.smithi042.hljzvq on smithi042 2024-09-13T13:01:27.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:27 smithi046 ceph-mon[55972]: from='client.44101 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:28.228 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:01:28.228 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:01:28.228 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:01:28.228 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:01:28.228 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 4 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:01:28.229 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:01:28.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 134445 9.63 MB/sec execute 213 sec latency 44.125 ms 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: pgmap v25: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:28.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:28 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/518058675' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:01:28.770 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:01:28.770 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:01:28.770 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:01:28.770 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:01:28.770 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "4/23 daemons upgraded", 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mon daemons", 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:01:28.771 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:01:28.862 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='client.34156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: pgmap v25: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='client.44109 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:28.863 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:28 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/518058675' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:01:29.457 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:01:29.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 134785 9.65 MB/sec execute 214 sec latency 31.102 ms 2024-09-13T13:01:29.655 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:29 smithi042 ceph-mon[63970]: Reconfiguring crash.smithi046 (monmap changed)... 2024-09-13T13:01:29.655 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:29 smithi042 ceph-mon[63970]: Reconfiguring daemon crash.smithi046 on smithi046 2024-09-13T13:01:29.655 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:29.655 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:29 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1610648009' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:01:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:29 smithi046 ceph-mon[55972]: Reconfiguring crash.smithi046 (monmap changed)... 2024-09-13T13:01:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:29 smithi046 ceph-mon[55972]: Reconfiguring daemon crash.smithi046 on smithi046 2024-09-13T13:01:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:29 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1610648009' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:01:30.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 135342 9.66 MB/sec execute 215 sec latency 63.784 ms 2024-09-13T13:01:30.664 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:30.664 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: pgmap v26: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-13T13:01:30.664 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:30.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:30.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: Reconfiguring mgr.smithi046.hcgddm (monmap changed)... 2024-09-13T13:01:30.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T13:01:30.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T13:01:30.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:30.665 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:30 smithi042 ceph-mon[63970]: Reconfiguring daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T13:01:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: from='client.34168 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:01:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: pgmap v26: 97 pgs: 97 active+clean; 344 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.2 MiB/s wr, 414 op/s 2024-09-13T13:01:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:31.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: Reconfiguring mgr.smithi046.hcgddm (monmap changed)... 2024-09-13T13:01:31.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mgr.smithi046.hcgddm", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-13T13:01:31.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mgr services"} : dispatch 2024-09-13T13:01:31.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:31.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:30 smithi046 ceph-mon[55972]: Reconfiguring daemon mgr.smithi046.hcgddm on smithi046 2024-09-13T13:01:31.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 135732 9.70 MB/sec execute 216 sec latency 43.869 ms 2024-09-13T13:01:32.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 136116 9.69 MB/sec execute 217 sec latency 54.333 ms 2024-09-13T13:01:32.682 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 638 op/s 2024-09-13T13:01:32.682 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:32.682 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:32.682 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: Reconfiguring mon.smithi046 (monmap changed)... 2024-09-13T13:01:32.682 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T13:01:32.683 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T13:01:32.683 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:32.683 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:32 smithi042 ceph-mon[63970]: Reconfiguring daemon mon.smithi046 on smithi046 2024-09-13T13:01:32.984 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: pgmap v27: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 9.7 KiB/s rd, 6.3 MiB/s wr, 638 op/s 2024-09-13T13:01:32.984 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:32.984 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:32.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: Reconfiguring mon.smithi046 (monmap changed)... 2024-09-13T13:01:32.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-13T13:01:32.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-13T13:01:32.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:32.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:32 smithi046 ceph-mon[55972]: Reconfiguring daemon mon.smithi046 on smithi046 2024-09-13T13:01:33.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 136470 9.68 MB/sec execute 218 sec latency 96.911 ms 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.4 on smithi046 2024-09-13T13:01:34.220 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:34 smithi042 ceph-mon[63970]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 452 op/s 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: Reconfiguring osd.4 (monmap changed)... 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.4 on smithi046 2024-09-13T13:01:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:34 smithi046 ceph-mon[55972]: pgmap v28: 97 pgs: 97 active+clean; 341 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.6 MiB/s wr, 452 op/s 2024-09-13T13:01:34.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 136949 9.71 MB/sec execute 219 sec latency 69.259 ms 2024-09-13T13:01:35.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 137461 9.72 MB/sec execute 220 sec latency 38.631 ms 2024-09-13T13:01:36.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:36.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:36.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T13:01:36.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T13:01:36.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:36.004 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.5 on smithi046 2024-09-13T13:01:36.004 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:35 smithi042 ceph-mon[63970]: pgmap v29: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 716 op/s 2024-09-13T13:01:36.273 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:36.273 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:36.273 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: Reconfiguring osd.5 (monmap changed)... 2024-09-13T13:01:36.274 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T13:01:36.274 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:36.274 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.5 on smithi046 2024-09-13T13:01:36.274 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:35 smithi046 ceph-mon[55972]: pgmap v29: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 716 op/s 2024-09-13T13:01:36.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 137870 9.70 MB/sec execute 221 sec latency 88.709 ms 2024-09-13T13:01:37.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 138529 9.76 MB/sec execute 222 sec latency 44.991 ms 2024-09-13T13:01:38.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: pgmap v30: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-13T13:01:38.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:38.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:38.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T13:01:38.295 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T13:01:38.296 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:38.296 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:38 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.6 on smithi046 2024-09-13T13:01:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: pgmap v30: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-13T13:01:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: Reconfiguring osd.6 (monmap changed)... 2024-09-13T13:01:38.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T13:01:38.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:38.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:38 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.6 on smithi046 2024-09-13T13:01:38.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 139021 9.78 MB/sec execute 223 sec latency 41.409 ms 2024-09-13T13:01:39.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 139332 9.78 MB/sec execute 224 sec latency 49.913 ms 2024-09-13T13:01:39.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:39.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:39.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T13:01:39.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T13:01:39.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:39.818 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: Reconfiguring daemon osd.7 on smithi046 2024-09-13T13:01:39.819 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:39 smithi042 ceph-mon[63970]: pgmap v31: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: Reconfiguring osd.7 (monmap changed)... 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: Reconfiguring daemon osd.7 on smithi046 2024-09-13T13:01:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:39 smithi046 ceph-mon[55972]: pgmap v31: 97 pgs: 97 active+clean; 346 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 5.0 MiB/s wr, 488 op/s 2024-09-13T13:01:40.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 139609 9.77 MB/sec execute 225 sec latency 67.311 ms 2024-09-13T13:01:41.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:41 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:41.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:41 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:41.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:41 smithi042 ceph-mon[63970]: Reconfiguring mds.foofs.smithi046.nqnolk (monmap changed)... 2024-09-13T13:01:41.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:41 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:01:41.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:41 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:41.602 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:41 smithi042 ceph-mon[63970]: Reconfiguring daemon mds.foofs.smithi046.nqnolk on smithi046 2024-09-13T13:01:41.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 139878 9.75 MB/sec execute 226 sec latency 66.177 ms 2024-09-13T13:01:41.755 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:41 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:41.755 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:41 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:41.755 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:41 smithi046 ceph-mon[55972]: Reconfiguring mds.foofs.smithi046.nqnolk (monmap changed)... 2024-09-13T13:01:41.755 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:41 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:01:41.755 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:41 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:41.755 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:41 smithi046 ceph-mon[55972]: Reconfiguring daemon mds.foofs.smithi046.nqnolk on smithi046 2024-09-13T13:01:42.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 140162 9.73 MB/sec execute 227 sec latency 54.026 ms 2024-09-13T13:01:42.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: pgmap v32: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 766 op/s 2024-09-13T13:01:42.633 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi042"} : dispatch 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi042"}]': finished 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi046"} : dispatch 2024-09-13T13:01:42.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi046"}]': finished 2024-09-13T13:01:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: pgmap v32: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.8 MiB/s wr, 766 op/s 2024-09-13T13:01:42.902 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:42.902 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi042"} : dispatch 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi042"}]': finished 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.smithi046"} : dispatch 2024-09-13T13:01:42.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.smithi046"}]': finished 2024-09-13T13:01:43.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 140544 9.75 MB/sec execute 228 sec latency 52.943 ms 2024-09-13T13:01:43.639 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:43 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all mon 2024-09-13T13:01:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:43 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all mon 2024-09-13T13:01:44.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 141052 9.75 MB/sec execute 229 sec latency 37.833 ms 2024-09-13T13:01:44.657 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:44 smithi042 ceph-mon[63970]: pgmap v33: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-09-13T13:01:44.657 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:44 smithi046 ceph-mon[55972]: pgmap v33: 97 pgs: 97 active+clean; 350 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.7 MiB/s wr, 542 op/s 2024-09-13T13:01:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:45.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 141421 9.73 MB/sec execute 230 sec latency 53.524 ms 2024-09-13T13:01:45.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:45 smithi042 ceph-mon[63970]: Upgrade: Updating crash.smithi042 (1/2) 2024-09-13T13:01:45.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:45 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:45.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:45 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:45.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:45 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:45.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:45 smithi042 ceph-mon[63970]: Deploying daemon crash.smithi042 on smithi042 2024-09-13T13:01:45.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:45 smithi042 ceph-mon[63970]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 748 op/s 2024-09-13T13:01:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:45 smithi046 ceph-mon[55972]: Upgrade: Updating crash.smithi042 (1/2) 2024-09-13T13:01:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:45 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:45 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi042", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:45 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:45 smithi046 ceph-mon[55972]: Deploying daemon crash.smithi042 on smithi042 2024-09-13T13:01:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:45 smithi046 ceph-mon[55972]: pgmap v34: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 748 op/s 2024-09-13T13:01:46.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 141688 9.73 MB/sec execute 231 sec latency 50.371 ms 2024-09-13T13:01:47.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 142013 9.74 MB/sec execute 232 sec latency 106.093 ms 2024-09-13T13:01:48.524 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:48 smithi042 ceph-mon[63970]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-09-13T13:01:48.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:48 smithi046 ceph-mon[55972]: pgmap v35: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-09-13T13:01:48.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 142305 9.72 MB/sec execute 233 sec latency 92.801 ms 2024-09-13T13:01:49.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 142514 9.72 MB/sec execute 234 sec latency 90.334 ms 2024-09-13T13:01:50.345 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:50 smithi042 ceph-mon[63970]: pgmap v36: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-09-13T13:01:50.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:50 smithi046 ceph-mon[55972]: pgmap v36: 97 pgs: 97 active+clean; 340 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 4.8 MiB/s wr, 484 op/s 2024-09-13T13:01:50.620 INFO:teuthology.orchestra.run.smithi042.stdout: 5 142712 9.71 MB/sec execute 235 sec latency 153.003 ms 2024-09-13T13:01:51.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 142984 9.71 MB/sec execute 236 sec latency 69.734 ms 2024-09-13T13:01:51.629 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:51.629 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:51.809 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:51.809 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:52.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:52 smithi046 ceph-mon[55972]: pgmap v37: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 678 op/s 2024-09-13T13:01:52.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:52.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:52.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:52.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 143233 9.68 MB/sec execute 237 sec latency 54.002 ms 2024-09-13T13:01:52.673 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:52 smithi042 ceph-mon[63970]: pgmap v37: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.8 MiB/s wr, 678 op/s 2024-09-13T13:01:52.673 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:52.673 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.crash.smithi046", "caps": ["mon", "profile crash", "mgr", "profile crash"]} : dispatch 2024-09-13T13:01:52.673 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:01:53.440 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:53 smithi042 ceph-mon[63970]: Upgrade: Updating crash.smithi046 (2/2) 2024-09-13T13:01:53.440 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:53 smithi042 ceph-mon[63970]: Deploying daemon crash.smithi046 on smithi046 2024-09-13T13:01:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:53 smithi046 ceph-mon[55972]: Upgrade: Updating crash.smithi046 (2/2) 2024-09-13T13:01:53.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:53 smithi046 ceph-mon[55972]: Deploying daemon crash.smithi046 on smithi046 2024-09-13T13:01:53.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 143458 9.66 MB/sec execute 238 sec latency 82.103 ms 2024-09-13T13:01:54.488 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:54 smithi042 ceph-mon[63970]: pgmap v38: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.0 MiB/s wr, 400 op/s 2024-09-13T13:01:54.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:54 smithi046 ceph-mon[55972]: pgmap v38: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.0 MiB/s wr, 400 op/s 2024-09-13T13:01:54.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 143709 9.64 MB/sec execute 239 sec latency 49.740 ms 2024-09-13T13:01:55.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 143997 9.64 MB/sec execute 240 sec latency 43.044 ms 2024-09-13T13:01:56.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 144386 9.65 MB/sec execute 241 sec latency 42.919 ms 2024-09-13T13:01:56.786 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:56 smithi042 ceph-mon[63970]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 537 op/s 2024-09-13T13:01:56.786 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:56.786 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:56.787 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:01:57.019 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:56 smithi046 ceph-mon[55972]: pgmap v39: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.4 MiB/s wr, 537 op/s 2024-09-13T13:01:57.019 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:57.019 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:57.020 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:01:57.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 144835 9.64 MB/sec execute 242 sec latency 39.546 ms 2024-09-13T13:01:58.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 145105 9.62 MB/sec execute 243 sec latency 54.100 ms 2024-09-13T13:01:58.908 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:58 smithi042 ceph-mon[63970]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 331 op/s 2024-09-13T13:01:59.086 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:58 smithi046 ceph-mon[55972]: pgmap v40: 97 pgs: 97 active+clean; 347 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.4 MiB/s wr, 331 op/s 2024-09-13T13:01:59.621 INFO:teuthology.orchestra.run.smithi042.stdout: 5 145430 9.62 MB/sec execute 244 sec latency 43.400 ms 2024-09-13T13:01:59.677 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:01:59.677 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:01:59.677 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:01:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:00.024 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:02:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:01:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:00.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 145844 9.64 MB/sec execute 245 sec latency 45.872 ms 2024-09-13T13:02:00.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:00 smithi042 ceph-mon[63970]: pgmap v41: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 340 op/s 2024-09-13T13:02:00.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:00 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:00.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:00 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:00.949 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:00 smithi046 ceph-mon[55972]: pgmap v41: 97 pgs: 97 active+clean; 346 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.5 MiB/s wr, 340 op/s 2024-09-13T13:02:00.949 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:00 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:00.950 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:00 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (3m) 68s ago 15m 21.9M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 starting - - - - 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (6s) 1s ago 12m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (3m) 68s ago 12m 43.8M - dad864ee21e9 7144014c89b8 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (6m) 68s ago 6m 40.5M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (6m) 1s ago 6m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (2m) 68s ago 16m 488M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (109s) 1s ago 12m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (91s) 68s ago 17m 34.4M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (61s) 1s ago 12m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:02:01.246 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (6m) 68s ago 6m 262M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (6m) 1s ago 6m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (3m) 68s ago 12m 9773k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (2m) 1s ago 11m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (11m) 68s ago 11m 461M 1937M 17.2.0 e1d6a67b021e 83b85e7cfcdc 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (10m) 68s ago 10m 398M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (10m) 68s ago 10m 264M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (9m) 68s ago 9m 413M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (9m) 1s ago 9m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (9m) 1s ago 9m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (8m) 1s ago 8m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (8m) 1s ago 8m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:02:01.247 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (115s) 68s ago 11m 44.1M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:02:01.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 146302 9.68 MB/sec execute 246 sec latency 35.222 ms 2024-09-13T13:02:01.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:01 smithi042 ceph-mon[63970]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:01.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:01 smithi042 ceph-mon[63970]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:01.974 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:02:01.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:02:01.974 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:02:01.974 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:01.974 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:02:01.974 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 4 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:02:01.975 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:02:01.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:01 smithi046 ceph-mon[55972]: from='client.34176 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:01.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:01 smithi046 ceph-mon[55972]: from='client.34180 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr" 2024-09-13T13:02:02.574 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:02:02.575 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "5/23 daemons upgraded", 2024-09-13T13:02:02.575 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading crash daemons", 2024-09-13T13:02:02.575 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:02:02.575 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:02:02.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 146646 9.67 MB/sec execute 247 sec latency 42.076 ms 2024-09-13T13:02:02.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:02 smithi042 ceph-mon[63970]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:02.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:02 smithi042 ceph-mon[63970]: pgmap v42: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 550 op/s 2024-09-13T13:02:02.708 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:02 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/846372700' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:03.009 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:02 smithi046 ceph-mon[55972]: from='client.44131 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:03.009 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:02 smithi046 ceph-mon[55972]: pgmap v42: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 5.4 MiB/s wr, 550 op/s 2024-09-13T13:02:03.010 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:02 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/846372700' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:03.312 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:02:03.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 146974 9.66 MB/sec execute 248 sec latency 65.277 ms 2024-09-13T13:02:03.720 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:03 smithi042 ceph-mon[63970]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:03.720 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:03 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/169900179' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:02:03.947 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:03 smithi046 ceph-mon[55972]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:03.948 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:03 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/169900179' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:02:04.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 147294 9.65 MB/sec execute 249 sec latency 38.810 ms 2024-09-13T13:02:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:04 smithi042 ceph-mon[63970]: pgmap v43: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 355 op/s 2024-09-13T13:02:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:04 smithi046 ceph-mon[55972]: pgmap v43: 97 pgs: 97 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.3 MiB/s wr, 355 op/s 2024-09-13T13:02:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:05.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 147591 9.64 MB/sec execute 250 sec latency 47.252 ms 2024-09-13T13:02:06.007 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:05 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:06.007 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:05 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:06.007 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:05 smithi042 ceph-mon[63970]: pgmap v44: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-09-13T13:02:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:05 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:05 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:05 smithi046 ceph-mon[55972]: pgmap v44: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.8 MiB/s wr, 574 op/s 2024-09-13T13:02:06.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 148045 9.66 MB/sec execute 251 sec latency 46.805 ms 2024-09-13T13:02:07.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 148668 9.67 MB/sec execute 252 sec latency 48.141 ms 2024-09-13T13:02:08.501 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:08 smithi046 ceph-mon[55972]: pgmap v45: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 437 op/s 2024-09-13T13:02:08.531 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:08 smithi042 ceph-mon[63970]: pgmap v45: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 4.5 MiB/s wr, 437 op/s 2024-09-13T13:02:08.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 149400 9.71 MB/sec execute 253 sec latency 24.244 ms 2024-09-13T13:02:09.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 149959 9.75 MB/sec execute 254 sec latency 39.252 ms 2024-09-13T13:02:10.301 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:10 smithi042 ceph-mon[63970]: pgmap v46: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 466 op/s 2024-09-13T13:02:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:10 smithi046 ceph-mon[55972]: pgmap v46: 97 pgs: 97 active+clean; 345 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.0 MiB/s wr, 466 op/s 2024-09-13T13:02:10.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 150428 9.76 MB/sec execute 255 sec latency 36.842 ms 2024-09-13T13:02:11.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 150793 9.75 MB/sec execute 256 sec latency 59.637 ms 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: pgmap v47: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 740 op/s 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.573 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi042"} : dispatch 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi042"}]': finished 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi046"} : dispatch 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi046"}]': finished 2024-09-13T13:02:12.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T13:02:12.622 INFO:teuthology.orchestra.run.smithi042.stdout: 5 151124 9.75 MB/sec execute 257 sec latency 39.372 ms 2024-09-13T13:02:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: pgmap v47: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.5 MiB/s wr, 740 op/s 2024-09-13T13:02:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi042"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi042"}]': finished 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi046"} : dispatch 2024-09-13T13:02:12.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash.smithi046"}]': finished 2024-09-13T13:02:12.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-13T13:02:13.588 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:13 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all crash 2024-09-13T13:02:13.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:13 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T13:02:13.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:13 smithi042 ceph-mon[63970]: Upgrade: osd.0 is safe to restart 2024-09-13T13:02:13.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 151593 9.77 MB/sec execute 258 sec latency 44.228 ms 2024-09-13T13:02:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:13 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all crash 2024-09-13T13:02:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:13 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T13:02:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:13 smithi046 ceph-mon[55972]: Upgrade: osd.0 is safe to restart 2024-09-13T13:02:14.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:14 smithi042 ceph-mon[63970]: pgmap v48: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 530 op/s 2024-09-13T13:02:14.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:14.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 152308 9.78 MB/sec execute 259 sec latency 25.653 ms 2024-09-13T13:02:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:14 smithi046 ceph-mon[55972]: pgmap v48: 97 pgs: 97 active+clean; 352 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.7 MiB/s wr, 530 op/s 2024-09-13T13:02:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:15.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 152957 9.82 MB/sec execute 260 sec latency 47.836 ms 2024-09-13T13:02:15.851 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:15 smithi042 ceph-mon[63970]: Upgrade: Updating osd.0 2024-09-13T13:02:15.851 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:15.851 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T13:02:15.851 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:02:15.851 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:15 smithi042 ceph-mon[63970]: Deploying daemon osd.0 on smithi042 2024-09-13T13:02:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:15 smithi046 ceph-mon[55972]: Upgrade: Updating osd.0 2024-09-13T13:02:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-13T13:02:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:02:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:15 smithi046 ceph-mon[55972]: Deploying daemon osd.0 on smithi042 2024-09-13T13:02:16.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 153363 9.84 MB/sec execute 261 sec latency 49.101 ms 2024-09-13T13:02:16.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:16 smithi042 ceph-mon[63970]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 810 op/s 2024-09-13T13:02:16.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:16 smithi042 ceph-mon[63970]: mgrmap e43: smithi042.jhsfyp(active, since 92s), standbys: smithi046.hcgddm 2024-09-13T13:02:17.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:16 smithi046 ceph-mon[55972]: pgmap v49: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 8.3 MiB/s wr, 810 op/s 2024-09-13T13:02:17.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:16 smithi046 ceph-mon[55972]: mgrmap e43: smithi042.jhsfyp(active, since 92s), standbys: smithi046.hcgddm 2024-09-13T13:02:17.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 153728 9.85 MB/sec execute 262 sec latency 48.246 ms 2024-09-13T13:02:18.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 153941 9.82 MB/sec execute 263 sec latency 65.413 ms 2024-09-13T13:02:18.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:18 smithi042 ceph-mon[63970]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 592 op/s 2024-09-13T13:02:19.040 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:18 smithi046 ceph-mon[55972]: pgmap v50: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.8 MiB/s wr, 592 op/s 2024-09-13T13:02:19.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 154217 9.81 MB/sec execute 264 sec latency 50.578 ms 2024-09-13T13:02:19.975 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:19 smithi042 systemd[1]: Stopping Ceph osd.0 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:02:20.242 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:20 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[34384]: 2024-09-13T13:02:20.108+0000 7fa7eadea700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:02:20.242 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:20 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[34384]: 2024-09-13T13:02:20.108+0000 7fa7eadea700 -1 osd.0 73 *** Got signal Terminated *** 2024-09-13T13:02:20.242 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:20 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[34384]: 2024-09-13T13:02:20.108+0000 7fa7eadea700 -1 osd.0 73 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:02:20.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 154536 9.81 MB/sec execute 265 sec latency 52.831 ms 2024-09-13T13:02:20.749 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:20 smithi042 ceph-mon[63970]: pgmap v51: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 675 op/s 2024-09-13T13:02:20.749 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:20 smithi042 ceph-mon[63970]: osd.0 marked itself down and dead 2024-09-13T13:02:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:20 smithi046 ceph-mon[55972]: pgmap v51: 97 pgs: 97 active+clean; 344 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 6.5 MiB/s wr, 675 op/s 2024-09-13T13:02:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:20 smithi046 ceph-mon[55972]: osd.0 marked itself down and dead 2024-09-13T13:02:21.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 154737 9.79 MB/sec execute 266 sec latency 912.596 ms 2024-09-13T13:02:21.630 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:21 smithi042 podman[68303]: 2024-09-13 13:02:21.320992494 +0000 UTC m=+1.373369504 container died 83b85e7cfcdcfcddc715fdecf0a7eb5c701afb53dac86a3ad1ed9e565d90e0bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, release=754, vcs-type=git, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, architecture=x86_64, io.openshift.expose-services=, name=centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., distribution-scope=public) 2024-09-13T13:02:21.881 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:21 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:02:21.881 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:21 smithi042 ceph-mon[63970]: osdmap e74: 8 total, 7 up, 8 in 2024-09-13T13:02:21.881 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:21 smithi042 podman[68303]: 2024-09-13 13:02:21.736917927 +0000 UTC m=+1.789294946 container cleanup 83b85e7cfcdcfcddc715fdecf0a7eb5c701afb53dac86a3ad1ed9e565d90e0bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, architecture=x86_64, RELEASE=HEAD) 2024-09-13T13:02:21.882 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:21 smithi042 bash[68303]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0 2024-09-13T13:02:22.106 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:22 smithi042 podman[68320]: 2024-09-13 13:02:22.066973321 +0000 UTC m=+0.737592197 container remove 83b85e7cfcdcfcddc715fdecf0a7eb5c701afb53dac86a3ad1ed9e565d90e0bd (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, vcs-type=git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, io.openshift.tags=base centos centos-stream) 2024-09-13T13:02:22.116 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:21 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:02:22.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:21 smithi046 ceph-mon[55972]: osdmap e74: 8 total, 7 up, 8 in 2024-09-13T13:02:22.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 155404 9.81 MB/sec execute 267 sec latency 1006.435 ms 2024-09-13T13:02:22.888 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:22 smithi042 ceph-mon[63970]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 631 op/s 2024-09-13T13:02:22.888 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:22 smithi042 ceph-mon[63970]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T13:02:22.889 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:22 smithi042 podman[68401]: 2024-09-13 13:02:22.560704958 +0000 UTC m=+0.025626262 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:02:22.889 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:22 smithi042 podman[68401]: 2024-09-13 13:02:22.7079792 +0000 UTC m=+0.172900496 container create 2a3f781c031e1143c720d93df5e2e88dcef92bf4e104ce5c5a4fbc6042e223af (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:02:23.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:22 smithi046 ceph-mon[55972]: pgmap v53: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 6.4 MiB/s wr, 631 op/s 2024-09-13T13:02:23.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:22 smithi046 ceph-mon[55972]: osdmap e75: 8 total, 7 up, 8 in 2024-09-13T13:02:23.394 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:23 smithi042 podman[68401]: 2024-09-13 13:02:23.162944847 +0000 UTC m=+0.627866146 container init 2a3f781c031e1143c720d93df5e2e88dcef92bf4e104ce5c5a4fbc6042e223af (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T13:02:23.394 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:23 smithi042 podman[68401]: 2024-09-13 13:02:23.170647439 +0000 UTC m=+0.635568733 container start 2a3f781c031e1143c720d93df5e2e88dcef92bf4e104ce5c5a4fbc6042e223af (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2) 2024-09-13T13:02:23.394 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:23 smithi042 podman[68401]: 2024-09-13 13:02:23.349381502 +0000 UTC m=+0.814302788 container attach 2a3f781c031e1143c720d93df5e2e88dcef92bf4e104ce5c5a4fbc6042e223af (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-13T13:02:23.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 156217 9.84 MB/sec execute 268 sec latency 25.818 ms 2024-09-13T13:02:23.637 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:23 smithi042 podman[68401]: 2024-09-13 13:02:23.536117564 +0000 UTC m=+1.001038859 container died 2a3f781c031e1143c720d93df5e2e88dcef92bf4e104ce5c5a4fbc6042e223af (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, ceph=True) 2024-09-13T13:02:24.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 156671 9.87 MB/sec execute 269 sec latency 53.093 ms 2024-09-13T13:02:24.644 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:24 smithi042 podman[68422]: 2024-09-13 13:02:24.595093358 +0000 UTC m=+1.041967189 container remove 2a3f781c031e1143c720d93df5e2e88dcef92bf4e104ce5c5a4fbc6042e223af (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:02:24.899 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:24 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T13:02:24.899 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:24 smithi042 systemd[1]: Stopped Ceph osd.0 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:02:24.900 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:24 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.0.service: Consumed 2min 40.331s CPU time. 2024-09-13T13:02:24.900 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:24 smithi042 ceph-mon[63970]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 369 op/s 2024-09-13T13:02:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:24 smithi046 ceph-mon[55972]: pgmap v55: 97 pgs: 13 stale+active+clean, 84 active+clean; 348 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 4.0 MiB/s wr, 369 op/s 2024-09-13T13:02:25.150 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:24 smithi042 systemd[1]: Starting Ceph osd.0 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:02:25.401 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 podman[68517]: 2024-09-13 13:02:25.102953002 +0000 UTC m=+0.023767496 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:02:25.401 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 podman[68517]: 2024-09-13 13:02:25.259530555 +0000 UTC m=+0.180345040 container create 88853b945c6b7bea750144bee69229e8b4540c7ae1989d13d25ab890818895d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T13:02:25.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 156991 9.88 MB/sec execute 270 sec latency 64.224 ms 2024-09-13T13:02:25.675 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 podman[68517]: 2024-09-13 13:02:25.630661023 +0000 UTC m=+0.551475509 container init 88853b945c6b7bea750144bee69229e8b4540c7ae1989d13d25ab890818895d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-13T13:02:25.675 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 podman[68517]: 2024-09-13 13:02:25.645884505 +0000 UTC m=+0.566698992 container start 88853b945c6b7bea750144bee69229e8b4540c7ae1989d13d25ab890818895d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T13:02:25.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:25 smithi042 ceph-mon[63970]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.5 MiB/s wr, 812 op/s; 237/1800 objects degraded (13.167%) 2024-09-13T13:02:25.927 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:25 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 237/1800 objects degraded (13.167%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:25.927 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 podman[68517]: 2024-09-13 13:02:25.781377679 +0000 UTC m=+0.702192175 container attach 88853b945c6b7bea750144bee69229e8b4540c7ae1989d13d25ab890818895d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True) 2024-09-13T13:02:25.927 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:02:25.928 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 bash[68517]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:02:25.928 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:02:25.928 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:25 smithi042 bash[68517]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:02:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:25 smithi046 ceph-mon[55972]: pgmap v56: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.6 KiB/s rd, 8.5 MiB/s wr, 812 op/s; 237/1800 objects degraded (13.167%) 2024-09-13T13:02:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:25 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 237/1800 objects degraded (13.167%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:26.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 157303 9.88 MB/sec execute 271 sec latency 75.091 ms 2024-09-13T13:02:27.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 157590 9.86 MB/sec execute 272 sec latency 41.130 ms 2024-09-13T13:02:28.477 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-mon[63970]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 7.5 MiB/s wr, 688 op/s; 237/1800 objects degraded (13.167%) 2024-09-13T13:02:28.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:28 smithi046 ceph-mon[55972]: pgmap v57: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 7.5 MiB/s wr, 688 op/s; 237/1800 objects degraded (13.167%) 2024-09-13T13:02:28.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 157839 9.84 MB/sec execute 273 sec latency 103.655 ms 2024-09-13T13:02:28.731 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T13:02:28.731 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T13:02:28.731 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T13:02:28.731 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T13:02:28.994 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T13:02:28.994 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T13:02:28.995 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T13:02:28.995 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T13:02:28.995 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T13:02:28.995 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T13:02:28.995 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T13:02:28.995 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T13:02:29.246 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate[68531]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T13:02:29.246 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:28 smithi042 bash[68517]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T13:02:29.246 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:29 smithi042 podman[68517]: 2024-09-13 13:02:29.034638152 +0000 UTC m=+3.955452648 container died 88853b945c6b7bea750144bee69229e8b4540c7ae1989d13d25ab890818895d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-13T13:02:29.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 158129 9.83 MB/sec execute 274 sec latency 58.381 ms 2024-09-13T13:02:30.010 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:29 smithi042 podman[68689]: 2024-09-13 13:02:29.879573746 +0000 UTC m=+0.837957725 container remove 88853b945c6b7bea750144bee69229e8b4540c7ae1989d13d25ab890818895d1 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-activate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS) 2024-09-13T13:02:30.010 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:30.010 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:30.010 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:29 smithi042 ceph-mon[63970]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 531 op/s; 229/1761 objects degraded (13.004%) 2024-09-13T13:02:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:29 smithi046 ceph-mon[55972]: pgmap v58: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.2 MiB/s wr, 531 op/s; 229/1761 objects degraded (13.004%) 2024-09-13T13:02:30.273 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 podman[68748]: 2024-09-13 13:02:30.014688076 +0000 UTC m=+0.021010258 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:02:30.273 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 podman[68748]: 2024-09-13 13:02:30.147465522 +0000 UTC m=+0.153787699 container create 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:02:30.558 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 podman[68748]: 2024-09-13 13:02:30.433954619 +0000 UTC m=+0.440276799 container init 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T13:02:30.558 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 podman[68748]: 2024-09-13 13:02:30.444972547 +0000 UTC m=+0.451294725 container start 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T13:02:30.558 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 ceph-osd[68766]: -- 172.21.15.42:0/894688146 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5589057881e0 con 0x558905714000 2024-09-13T13:02:30.558 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 bash[68748]: 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 2024-09-13T13:02:30.558 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:30 smithi042 systemd[1]: Started Ceph osd.0 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:02:30.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 158496 9.84 MB/sec execute 275 sec latency 55.588 ms 2024-09-13T13:02:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 158836 9.82 MB/sec execute 276 sec latency 58.338 ms 2024-09-13T13:02:31.847 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:31.847 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:31.847 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:02:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:02:32.615 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:32 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:02:32.485+0000 7f7f47e9f640 -1 Falling back to public interface 2024-09-13T13:02:32.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 159102 9.81 MB/sec execute 277 sec latency 100.449 ms 2024-09-13T13:02:32.866 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:32 smithi042 ceph-mon[63970]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.0 MiB/s wr, 579 op/s; 214/1719 objects degraded (12.449%) 2024-09-13T13:02:32.866 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:32 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 214/1719 objects degraded (12.449%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:32 smithi046 ceph-mon[55972]: pgmap v59: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 6.0 MiB/s wr, 579 op/s; 214/1719 objects degraded (12.449%) 2024-09-13T13:02:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:32 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 214/1719 objects degraded (12.449%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:33.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 159413 9.81 MB/sec execute 278 sec latency 64.639 ms 2024-09-13T13:02:33.809 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:02:34.623 INFO:teuthology.orchestra.run.smithi042.stdout: 5 159761 9.80 MB/sec execute 279 sec latency 63.446 ms 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (4m) 30s ago 16m 21.9M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (44s) 30s ago 16m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (39s) 35s ago 12m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (3m) 30s ago 13m 43.7M - dad864ee21e9 7144014c89b8 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (7m) 30s ago 7m 40.6M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (7m) 35s ago 7m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (3m) 30s ago 17m 509M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (2m) 35s ago 12m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:02:34.739 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (2m) 30s ago 17m 46.8M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (94s) 35s ago 12m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (6m) 30s ago 6m 264M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (6m) 35s ago 6m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (3m) 30s ago 13m 9911k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (3m) 35s ago 12m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 starting - - - 1937M 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (11m) 30s ago 11m 450M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (10m) 30s ago 10m 306M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (10m) 30s ago 10m 480M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (10m) 35s ago 9m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (9m) 35s ago 9m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (9m) 35s ago 9m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:02:34.740 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (8m) 35s ago 8m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:02:34.741 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (2m) 30s ago 12m 50.7M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:02:34.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:34 smithi042 ceph-mon[63970]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 501 op/s; 214/1719 objects degraded (12.449%) 2024-09-13T13:02:35.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:34 smithi046 ceph-mon[55972]: pgmap v60: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 353 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 5.2 MiB/s wr, 501 op/s; 214/1719 objects degraded (12.449%) 2024-09-13T13:02:35.349 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:02:35.349 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:02:35.350 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:02:35.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:35.350 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:02:35.350 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:02:35.350 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:35.351 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:02:35.351 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7 2024-09-13T13:02:35.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:35.351 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:02:35.351 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:02:35.351 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:02:35.352 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:02:35.352 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 9, 2024-09-13T13:02:35.352 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 4 2024-09-13T13:02:35.352 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:02:35.352 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:02:35.631 INFO:teuthology.orchestra.run.smithi042.stdout: 5 160140 9.81 MB/sec execute 280 sec latency 55.877 ms 2024-09-13T13:02:35.754 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:35 smithi042 ceph-mon[63970]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:35.754 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:35 smithi042 ceph-mon[63970]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:35.754 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:35 smithi042 ceph-mon[63970]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:35.755 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:35 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1212220834' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:35.940 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "6/23 daemons upgraded", 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:02:35.941 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:02:35.942 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:02:36.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:35 smithi046 ceph-mon[55972]: from='client.34202 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:36.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:35 smithi046 ceph-mon[55972]: from='client.44151 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:36.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:35 smithi046 ceph-mon[55972]: from='client.34210 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:36.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:35 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1212220834' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:36.601 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 209/1611 objects degraded (12.973%), 29 pgs degraded 2024-09-13T13:02:36.601 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:02:36.601 INFO:teuthology.orchestra.run.smithi042.stdout: osd.0 (root=default,host=smithi042) is down 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 209/1611 objects degraded (12.973%), 29 pgs degraded 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 1.0 is active+undersized+degraded, acting [7,6] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.0 is active+undersized+degraded, acting [7,1] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1 is active+undersized+degraded, acting [2,3] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.4 is active+undersized+degraded, acting [1,7] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.5 is active+undersized+degraded, acting [7,4] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.8 is active+undersized+degraded, acting [7,1] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.c is active+undersized+degraded, acting [2,5] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.f is active+undersized+degraded, acting [4,7] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.10 is active+undersized+degraded, acting [2,1] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.13 is active+undersized+degraded, acting [4,3] 2024-09-13T13:02:36.602 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.15 is active+undersized+degraded, acting [1,5] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.19 is active+undersized+degraded, acting [3,6] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1d is active+undersized+degraded, acting [7,6] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1e is active+undersized+degraded, acting [3,5] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1f is active+undersized+degraded, acting [7,4] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1 is active+undersized+degraded, acting [4,3] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.3 is active+undersized+degraded, acting [4,6] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.6 is active+undersized+degraded, acting [1,4] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.7 is active+undersized+degraded, acting [3,7] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.b is active+undersized+degraded, acting [3,4] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.f is active+undersized+degraded, acting [7,4] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.10 is active+undersized+degraded, acting [6,5] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.12 is active+undersized+degraded, acting [7,3] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.17 is active+undersized+degraded, acting [5,3] 2024-09-13T13:02:36.603 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.18 is active+undersized+degraded, acting [3,1] 2024-09-13T13:02:36.604 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1b is active+undersized+degraded, acting [4,7] 2024-09-13T13:02:36.604 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1f is active+undersized+degraded, acting [5,2] 2024-09-13T13:02:36.604 INFO:teuthology.orchestra.run.smithi042.stdout: pg 4.0 is active+undersized+degraded, acting [3,7] 2024-09-13T13:02:36.604 INFO:teuthology.orchestra.run.smithi042.stdout: pg 4.19 is active+undersized+degraded, acting [3,2] 2024-09-13T13:02:36.631 INFO:teuthology.orchestra.run.smithi042.stdout: 5 160651 9.85 MB/sec execute 281 sec latency 43.661 ms 2024-09-13T13:02:36.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:36 smithi042 ceph-mon[63970]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.7 MiB/s wr, 680 op/s; 209/1611 objects degraded (12.973%) 2024-09-13T13:02:36.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:36 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/902722426' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:02:37.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:36 smithi046 ceph-mon[55972]: pgmap v61: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 6.7 MiB/s wr, 680 op/s; 209/1611 objects degraded (12.973%) 2024-09-13T13:02:37.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:36 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/902722426' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:02:37.631 INFO:teuthology.orchestra.run.smithi042.stdout: 5 160974 9.83 MB/sec execute 282 sec latency 111.210 ms 2024-09-13T13:02:37.796 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:37 smithi042 ceph-mon[63970]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:37 smithi046 ceph-mon[55972]: from='client.34218 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:02:38.632 INFO:teuthology.orchestra.run.smithi042.stdout: 5 161256 9.82 MB/sec execute 283 sec latency 52.221 ms 2024-09-13T13:02:38.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:38 smithi042 ceph-mon[63970]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 385 op/s; 209/1611 objects degraded (12.973%) 2024-09-13T13:02:39.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:38 smithi046 ceph-mon[55972]: pgmap v62: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 343 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.7 MiB/s wr, 385 op/s; 209/1611 objects degraded (12.973%) 2024-09-13T13:02:39.614 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:39 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:02:39.358+0000 7f7f47e9f640 -1 osd.0 0 read_superblock omap replica is missing. 2024-09-13T13:02:39.631 INFO:teuthology.orchestra.run.smithi042.stdout: 5 161427 9.80 MB/sec execute 284 sec latency 70.252 ms 2024-09-13T13:02:39.784 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:39 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 214/1641 objects degraded (13.041%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:40.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:39 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 214/1641 objects degraded (13.041%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:40.632 INFO:teuthology.orchestra.run.smithi042.stdout: 5 161681 9.80 MB/sec execute 285 sec latency 84.531 ms 2024-09-13T13:02:40.637 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:40 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:02:40.445+0000 7f7f47e9f640 -1 osd.0 73 log_to_monitors true 2024-09-13T13:02:40.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:40 smithi042 ceph-mon[63970]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.5 MiB/s wr, 493 op/s; 214/1641 objects degraded (13.041%) 2024-09-13T13:02:40.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:40 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:40.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:40 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:40.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:40 smithi042 ceph-mon[63970]: from='osd.0 [v2:172.21.15.42:6802/2242789655,v1:172.21.15.42:6803/2242789655]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T13:02:40.910 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:40 smithi042 ceph-mon[63970]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T13:02:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:40 smithi046 ceph-mon[55972]: pgmap v63: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 347 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 4.5 MiB/s wr, 493 op/s; 214/1641 objects degraded (13.041%) 2024-09-13T13:02:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:40 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:40 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:40 smithi046 ceph-mon[55972]: from='osd.0 [v2:172.21.15.42:6802/2242789655,v1:172.21.15.42:6803/2242789655]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T13:02:41.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:40 smithi046 ceph-mon[55972]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-13T13:02:41.632 INFO:teuthology.orchestra.run.smithi042.stdout: 5 162057 9.79 MB/sec execute 286 sec latency 55.141 ms 2024-09-13T13:02:41.946 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:02:41 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:02:41.728+0000 7f7f3bc41640 -1 osd.0 73 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:02:42.204 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:42.204 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:42.204 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 512 op/s; 224/1701 objects degraded (13.169%) 2024-09-13T13:02:42.204 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T13:02:42.204 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: osdmap e76: 8 total, 7 up, 8 in 2024-09-13T13:02:42.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: from='osd.0 [v2:172.21.15.42:6802/2242789655,v1:172.21.15.42:6803/2242789655]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:02:42.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:02:42.205 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:42 smithi042 ceph-mon[63970]: from='osd.0 ' entity='osd.0' 2024-09-13T13:02:42.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:42.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:42.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: pgmap v64: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.3 MiB/s wr, 512 op/s; 224/1701 objects degraded (13.169%) 2024-09-13T13:02:42.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T13:02:42.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: osdmap e76: 8 total, 7 up, 8 in 2024-09-13T13:02:42.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: from='osd.0 [v2:172.21.15.42:6802/2242789655,v1:172.21.15.42:6803/2242789655]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:02:42.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:02:42.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:42 smithi046 ceph-mon[55972]: from='osd.0 ' entity='osd.0' 2024-09-13T13:02:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: 5 162545 9.80 MB/sec execute 287 sec latency 55.521 ms 2024-09-13T13:02:43.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:43 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:02:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:43 smithi042 ceph-mon[63970]: osd.0 [v2:172.21.15.42:6802/2242789655,v1:172.21.15.42:6803/2242789655] boot 2024-09-13T13:02:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:43 smithi042 ceph-mon[63970]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T13:02:43.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T13:02:43.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:43 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:02:43.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:43 smithi046 ceph-mon[55972]: osd.0 [v2:172.21.15.42:6802/2242789655,v1:172.21.15.42:6803/2242789655] boot 2024-09-13T13:02:43.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:43 smithi046 ceph-mon[55972]: osdmap e77: 8 total, 8 up, 8 in 2024-09-13T13:02:43.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-13T13:02:43.639 INFO:teuthology.orchestra.run.smithi042.stdout: 5 162705 9.78 MB/sec execute 288 sec latency 940.431 ms 2024-09-13T13:02:44.651 INFO:teuthology.orchestra.run.smithi042.stdout: 5 162705 9.75 MB/sec execute 289 sec latency 1940.485 ms 2024-09-13T13:02:44.963 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:44 smithi042 ceph-mon[63970]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.6 MiB/s wr, 316 op/s; 224/1701 objects degraded (13.169%) 2024-09-13T13:02:44.963 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:44 smithi042 ceph-mon[63970]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T13:02:44.963 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:44.963 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:44 smithi046 ceph-mon[55972]: pgmap v67: 97 pgs: 7 active+undersized, 29 active+undersized+degraded, 61 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.2 KiB/s rd, 3.6 MiB/s wr, 316 op/s; 224/1701 objects degraded (13.169%) 2024-09-13T13:02:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:44 smithi046 ceph-mon[55972]: osdmap e78: 8 total, 8 up, 8 in 2024-09-13T13:02:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:45.639 INFO:teuthology.orchestra.run.smithi042.stdout: 5 162899 9.73 MB/sec execute 290 sec latency 2085.349 ms 2024-09-13T13:02:45.990 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:45 smithi042 ceph-mon[63970]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T13:02:45.990 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:45 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 224/1701 objects degraded (13.169%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:45.990 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:45 smithi042 ceph-mon[63970]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T13:02:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:45 smithi046 ceph-mon[55972]: osdmap e79: 8 total, 8 up, 8 in 2024-09-13T13:02:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:45 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 224/1701 objects degraded (13.169%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:46.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:45 smithi046 ceph-mon[55972]: osdmap e80: 8 total, 8 up, 8 in 2024-09-13T13:02:46.648 INFO:teuthology.orchestra.run.smithi042.stdout: 5 163239 9.72 MB/sec execute 291 sec latency 74.292 ms 2024-09-13T13:02:46.746 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: pgmap v70: 97 pgs: 14 remapped+peering, 5 active+undersized, 1 active+undersized+degraded, 77 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 502 op/s; 6/1533 objects degraded (0.391%); 35 KiB/s, 0 objects/s recovering 2024-09-13T13:02:46.746 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:46.746 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:46.746 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:02:46.746 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:02:46.746 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:46.747 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:46.747 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:46 smithi042 ceph-mon[63970]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: pgmap v70: 97 pgs: 14 remapped+peering, 5 active+undersized, 1 active+undersized+degraded, 77 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 502 op/s; 6/1533 objects degraded (0.391%); 35 KiB/s, 0 objects/s recovering 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:46 smithi046 ceph-mon[55972]: osdmap e81: 8 total, 8 up, 8 in 2024-09-13T13:02:47.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 163510 9.72 MB/sec execute 292 sec latency 548.653 ms 2024-09-13T13:02:47.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:02:47.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:47.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:47.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:47.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T13:02:47.769 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:47 smithi042 ceph-mon[63970]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T13:02:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:02:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:02:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T13:02:48.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:47 smithi046 ceph-mon[55972]: osdmap e82: 8 total, 8 up, 8 in 2024-09-13T13:02:48.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 163790 9.71 MB/sec execute 293 sec latency 904.159 ms 2024-09-13T13:02:48.788 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:48 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T13:02:48.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:48 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T13:02:48.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:48 smithi042 ceph-mon[63970]: pgmap v73: 97 pgs: 1 peering, 13 remapped+peering, 83 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 502 op/s; 35 KiB/s, 0 objects/s recovering 2024-09-13T13:02:48.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:48 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1533 objects degraded (0.391%), 1 pg degraded) 2024-09-13T13:02:48.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:48 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:02:48.789 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:48 smithi042 ceph-mon[63970]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T13:02:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:48 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T13:02:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:48 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-09-13T13:02:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:48 smithi046 ceph-mon[55972]: pgmap v73: 97 pgs: 1 peering, 13 remapped+peering, 83 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 4.5 MiB/s wr, 502 op/s; 35 KiB/s, 0 objects/s recovering 2024-09-13T13:02:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:48 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 6/1533 objects degraded (0.391%), 1 pg degraded) 2024-09-13T13:02:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:48 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:02:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:48 smithi046 ceph-mon[55972]: osdmap e83: 8 total, 8 up, 8 in 2024-09-13T13:02:49.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 164050 9.71 MB/sec execute 294 sec latency 892.511 ms 2024-09-13T13:02:50.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:50 smithi042 ceph-mon[63970]: pgmap v76: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 84 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 417 op/s; 182/1560 objects degraded (11.667%); 1024 KiB/s, 12 objects/s recovering 2024-09-13T13:02:50.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:50 smithi042 ceph-mon[63970]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T13:02:50.598 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:50 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 182/1560 objects degraded (11.667%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:50.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 164282 9.70 MB/sec execute 295 sec latency 143.849 ms 2024-09-13T13:02:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:50 smithi046 ceph-mon[55972]: pgmap v76: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 84 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.0 MiB/s wr, 417 op/s; 182/1560 objects degraded (11.667%); 1024 KiB/s, 12 objects/s recovering 2024-09-13T13:02:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:50 smithi046 ceph-mon[55972]: osdmap e84: 8 total, 8 up, 8 in 2024-09-13T13:02:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:50 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 182/1560 objects degraded (11.667%), 8 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:51.611 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:51 smithi042 ceph-mon[63970]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T13:02:51.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 164595 9.69 MB/sec execute 296 sec latency 200.425 ms 2024-09-13T13:02:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:51 smithi046 ceph-mon[55972]: osdmap e85: 8 total, 8 up, 8 in 2024-09-13T13:02:52.624 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:52 smithi042 ceph-mon[63970]: pgmap v79: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 84 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 432 op/s; 182/1569 objects degraded (11.600%); 1024 KiB/s, 12 objects/s recovering 2024-09-13T13:02:52.624 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:52 smithi042 ceph-mon[63970]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T13:02:52.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 164860 9.68 MB/sec execute 297 sec latency 1020.472 ms 2024-09-13T13:02:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:52 smithi046 ceph-mon[55972]: pgmap v79: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 remapped+peering, 84 active+clean; 343 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.1 MiB/s wr, 432 op/s; 182/1569 objects degraded (11.600%); 1024 KiB/s, 12 objects/s recovering 2024-09-13T13:02:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:52 smithi046 ceph-mon[55972]: osdmap e86: 8 total, 8 up, 8 in 2024-09-13T13:02:53.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 165045 9.66 MB/sec execute 298 sec latency 158.222 ms 2024-09-13T13:02:53.693 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:53 smithi042 ceph-mon[63970]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T13:02:53.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:53 smithi046 ceph-mon[55972]: osdmap e87: 8 total, 8 up, 8 in 2024-09-13T13:02:54.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 165347 9.66 MB/sec execute 299 sec latency 123.600 ms 2024-09-13T13:02:54.706 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:54 smithi042 ceph-mon[63970]: pgmap v82: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 89 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 754 KiB/s wr, 37 op/s; 158/1578 objects degraded (10.013%); 1024 KiB/s, 23 objects/s recovering 2024-09-13T13:02:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:54 smithi046 ceph-mon[55972]: pgmap v82: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 peering, 89 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 754 KiB/s wr, 37 op/s; 158/1578 objects degraded (10.013%); 1024 KiB/s, 23 objects/s recovering 2024-09-13T13:02:55.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 165576 9.65 MB/sec execute 300 sec latency 65.370 ms 2024-09-13T13:02:55.731 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:55 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 158/1578 objects degraded (10.013%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:55.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:55 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 158/1578 objects degraded (10.013%), 7 pgs degraded (PG_DEGRADED) 2024-09-13T13:02:56.503 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:56 smithi042 ceph-mon[63970]: pgmap v83: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 338 op/s; 50/1605 objects degraded (3.115%); 702 KiB/s, 34 objects/s recovering 2024-09-13T13:02:56.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 165861 9.64 MB/sec execute 301 sec latency 67.564 ms 2024-09-13T13:02:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:56 smithi046 ceph-mon[55972]: pgmap v83: 97 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 344 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 3.2 MiB/s wr, 338 op/s; 50/1605 objects degraded (3.115%); 702 KiB/s, 34 objects/s recovering 2024-09-13T13:02:57.550 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:57 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 50/1605 objects degraded (3.115%), 2 pgs degraded) 2024-09-13T13:02:57.550 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:57 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:02:57.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 166153 9.64 MB/sec execute 302 sec latency 71.737 ms 2024-09-13T13:02:57.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:57 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 50/1605 objects degraded (3.115%), 2 pgs degraded) 2024-09-13T13:02:57.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:57 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:02:58.561 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:58 smithi042 ceph-mon[63970]: pgmap v84: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 307 op/s; 600 KiB/s, 37 objects/s recovering 2024-09-13T13:02:58.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 166512 9.64 MB/sec execute 303 sec latency 45.934 ms 2024-09-13T13:02:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:58 smithi046 ceph-mon[55972]: pgmap v84: 97 pgs: 97 active+clean; 342 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.2 MiB/s wr, 307 op/s; 600 KiB/s, 37 objects/s recovering 2024-09-13T13:02:59.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 166798 9.64 MB/sec execute 304 sec latency 62.704 ms 2024-09-13T13:02:59.864 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:02:59.864 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:02:59.864 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:02:59 smithi042 ceph-mon[63970]: pgmap v85: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 512 KiB/s, 31 objects/s recovering 2024-09-13T13:03:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:02:59 smithi046 ceph-mon[55972]: pgmap v85: 97 pgs: 97 active+clean; 349 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.2 MiB/s wr, 422 op/s; 512 KiB/s, 31 objects/s recovering 2024-09-13T13:03:00.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 167157 9.65 MB/sec execute 305 sec latency 45.712 ms 2024-09-13T13:03:01.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 167419 9.63 MB/sec execute 306 sec latency 59.015 ms 2024-09-13T13:03:02.329 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:02 smithi042 ceph-mon[63970]: pgmap v86: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 408 op/s; 0 B/s, 16 objects/s recovering 2024-09-13T13:03:02.329 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:02 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T13:03:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:02 smithi046 ceph-mon[55972]: pgmap v86: 97 pgs: 97 active+clean; 349 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 4.0 MiB/s wr, 408 op/s; 0 B/s, 16 objects/s recovering 2024-09-13T13:03:02.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:02 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-13T13:03:02.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 167668 9.61 MB/sec execute 307 sec latency 71.932 ms 2024-09-13T13:03:03.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:03 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T13:03:03.360 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:03 smithi042 ceph-mon[63970]: Upgrade: osd.1 is safe to restart 2024-09-13T13:03:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:03 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T13:03:03.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:03 smithi046 ceph-mon[55972]: Upgrade: osd.1 is safe to restart 2024-09-13T13:03:03.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 167851 9.60 MB/sec execute 308 sec latency 52.811 ms 2024-09-13T13:03:04.397 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:04 smithi042 ceph-mon[63970]: pgmap v87: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 381 op/s; 0 B/s, 14 objects/s recovering 2024-09-13T13:03:04.397 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:04.397 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:04.397 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T13:03:04.397 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:03:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:04 smithi046 ceph-mon[55972]: pgmap v87: 97 pgs: 97 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 3.9 MiB/s wr, 381 op/s; 0 B/s, 14 objects/s recovering 2024-09-13T13:03:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-13T13:03:04.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:03:04.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 168083 9.58 MB/sec execute 309 sec latency 63.425 ms 2024-09-13T13:03:05.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:05 smithi042 ceph-mon[63970]: Upgrade: Updating osd.1 2024-09-13T13:03:05.473 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:05 smithi042 ceph-mon[63970]: Deploying daemon osd.1 on smithi042 2024-09-13T13:03:05.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:05 smithi046 ceph-mon[55972]: Upgrade: Updating osd.1 2024-09-13T13:03:05.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:05 smithi046 ceph-mon[55972]: Deploying daemon osd.1 on smithi042 2024-09-13T13:03:05.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 168377 9.58 MB/sec execute 310 sec latency 74.270 ms 2024-09-13T13:03:06.478 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:06 smithi042 ceph-mon[63970]: pgmap v88: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 500 op/s; 0 B/s, 13 objects/s recovering 2024-09-13T13:03:06.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:06 smithi046 ceph-mon[55972]: pgmap v88: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 5.0 MiB/s wr, 500 op/s; 0 B/s, 13 objects/s recovering 2024-09-13T13:03:06.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 168759 9.58 MB/sec execute 311 sec latency 75.930 ms 2024-09-13T13:03:07.303 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:03:07.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 169087 9.58 MB/sec execute 312 sec latency 54.717 ms 2024-09-13T13:03:08.510 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:08 smithi042 ceph-mon[63970]: pgmap v89: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 357 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:03:08.510 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:08 smithi042 ceph-mon[63970]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:08.600 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:03:08.600 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (4m) 28s ago 16m 19.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:03:08.600 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (78s) 28s ago 16m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:03:08.600 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (73s) 69s ago 13m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (4m) 28s ago 13m 44.8M - dad864ee21e9 7144014c89b8 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (7m) 28s ago 7m 40.7M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (7m) 69s ago 7m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (3m) 28s ago 18m 511M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (2m) 69s ago 13m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (2m) 28s ago 18m 49.5M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (2m) 69s ago 13m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (7m) 28s ago 7m 265M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (7m) 69s ago 7m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (4m) 28s ago 13m 10.0M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (3m) 69s ago 13m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (38s) 28s ago 12m 12.2M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:03:08.601 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (11m) 28s ago 11m 483M 1937M 17.2.0 e1d6a67b021e 98e70e17880f 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (11m) 28s ago 11m 342M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (10m) 28s ago 10m 506M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (10m) 69s ago 10m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (10m) 69s ago 10m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (9m) 69s ago 9m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (9m) 69s ago 9m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:03:08.602 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (3m) 28s ago 12m 50.8M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:03:08.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:08 smithi046 ceph-mon[55972]: pgmap v89: 97 pgs: 97 active+clean; 346 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.3 KiB/s rd, 3.9 MiB/s wr, 357 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:03:08.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:08 smithi046 ceph-mon[55972]: from='client.34228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:08.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 169403 9.58 MB/sec execute 313 sec latency 45.540 ms 2024-09-13T13:03:08.766 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:08 smithi042 systemd[1]: Stopping Ceph osd.1 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:03:09.269 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:09 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[38296]: 2024-09-13T13:03:09.206+0000 7fb4a0fd9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:03:09.269 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:09 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[38296]: 2024-09-13T13:03:09.206+0000 7fb4a0fd9700 -1 osd.1 87 *** Got signal Terminated *** 2024-09-13T13:03:09.269 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:09 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[38296]: 2024-09-13T13:03:09.206+0000 7fb4a0fd9700 -1 osd.1 87 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:03:09.352 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:03:09.352 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:03:09.352 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:03:09.352 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:09.352 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:03:09.352 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 5 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:03:09.353 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:03:09.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:09 smithi046 ceph-mon[55972]: from='client.44171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:09.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:09 smithi046 ceph-mon[55972]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:09.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:09 smithi046 ceph-mon[55972]: osd.1 marked itself down and dead 2024-09-13T13:03:09.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 169651 9.57 MB/sec execute 314 sec latency 322.457 ms 2024-09-13T13:03:09.739 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:09 smithi042 ceph-mon[63970]: from='client.44171 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:09.739 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:09 smithi042 ceph-mon[63970]: from='client.34234 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:09.740 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:09 smithi042 ceph-mon[63970]: osd.1 marked itself down and dead 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:03:09.740 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:03:09.741 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:03:09.741 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "7/23 daemons upgraded", 2024-09-13T13:03:09.741 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:03:09.741 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:03:09.741 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:03:10.190 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down 2024-09-13T13:03:10.190 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:03:10.190 INFO:teuthology.orchestra.run.smithi042.stdout: osd.1 (root=default,host=smithi042) is down 2024-09-13T13:03:10.288 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:10 smithi042 podman[70762]: 2024-09-13 13:03:10.026825985 +0000 UTC m=+1.020367440 container died 98e70e17880fab23be067d980f39598c16116f9cf6df9dcfbf46f93f12c20769 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, vendor=Red Hat, Inc., RELEASE=HEAD, name=centos-stream, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.19.8) 2024-09-13T13:03:10.540 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:10 smithi042 podman[70762]: 2024-09-13 13:03:10.421277399 +0000 UTC m=+1.414818850 container cleanup 98e70e17880fab23be067d980f39598c16116f9cf6df9dcfbf46f93f12c20769 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , vcs-type=git, GIT_BRANCH=HEAD, RELEASE=HEAD, architecture=x86_64, vendor=Red Hat, Inc., ceph=True, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T13:03:10.541 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:10 smithi042 bash[70762]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1 2024-09-13T13:03:10.541 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:10 smithi042 ceph-mon[63970]: pgmap v90: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 512 op/s 2024-09-13T13:03:10.541 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:10 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:03:10.541 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:10 smithi042 ceph-mon[63970]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T13:03:10.541 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:10 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2264370249' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:10.541 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:10 smithi042 ceph-mon[63970]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:10.541 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:10 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/235149116' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:03:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:10 smithi046 ceph-mon[55972]: pgmap v90: 97 pgs: 97 active+clean; 351 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 5.1 MiB/s wr, 512 op/s 2024-09-13T13:03:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:10 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:03:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:10 smithi046 ceph-mon[55972]: osdmap e88: 8 total, 7 up, 8 in 2024-09-13T13:03:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:10 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2264370249' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:10 smithi046 ceph-mon[55972]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:10.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:10 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/235149116' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:03:10.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 169765 9.55 MB/sec execute 315 sec latency 1047.029 ms 2024-09-13T13:03:10.795 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:10 smithi042 podman[70822]: 2024-09-13 13:03:10.783665382 +0000 UTC m=+0.750659034 container remove 98e70e17880fab23be067d980f39598c16116f9cf6df9dcfbf46f93f12c20769 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, version=8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, ceph=True, RELEASE=HEAD, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True) 2024-09-13T13:03:11.550 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:11 smithi042 podman[70909]: 2024-09-13 13:03:11.285939078 +0000 UTC m=+0.023973611 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:03:11.550 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:11 smithi042 ceph-mon[63970]: osdmap e89: 8 total, 7 up, 8 in 2024-09-13T13:03:11.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:11 smithi046 ceph-mon[55972]: osdmap e89: 8 total, 7 up, 8 in 2024-09-13T13:03:11.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 170169 9.56 MB/sec execute 316 sec latency 31.164 ms 2024-09-13T13:03:11.803 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:11 smithi042 podman[70909]: 2024-09-13 13:03:11.604993806 +0000 UTC m=+0.343028322 container create 1416d0a7f7c38b777b1c257d7e73f5b168d56f9bb64deacb7e2f86bbcc2cb66f (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-13T13:03:12.307 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:12 smithi042 podman[70909]: 2024-09-13 13:03:12.16553634 +0000 UTC m=+0.903570858 container init 1416d0a7f7c38b777b1c257d7e73f5b168d56f9bb64deacb7e2f86bbcc2cb66f (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0) 2024-09-13T13:03:12.307 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:12 smithi042 podman[70909]: 2024-09-13 13:03:12.178568003 +0000 UTC m=+0.916602525 container start 1416d0a7f7c38b777b1c257d7e73f5b168d56f9bb64deacb7e2f86bbcc2cb66f (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:03:12.558 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:12 smithi042 ceph-mon[63970]: pgmap v93: 97 pgs: 14 stale+active+clean, 83 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.9 MiB/s wr, 518 op/s 2024-09-13T13:03:12.558 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:12 smithi042 podman[70909]: 2024-09-13 13:03:12.350493909 +0000 UTC m=+1.088528433 container attach 1416d0a7f7c38b777b1c257d7e73f5b168d56f9bb64deacb7e2f86bbcc2cb66f (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:03:12.558 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:12 smithi042 podman[70909]: 2024-09-13 13:03:12.540025725 +0000 UTC m=+1.278060250 container died 1416d0a7f7c38b777b1c257d7e73f5b168d56f9bb64deacb7e2f86bbcc2cb66f (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:03:12.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:12 smithi046 ceph-mon[55972]: pgmap v93: 97 pgs: 14 stale+active+clean, 83 active+clean; 347 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.9 KiB/s rd, 4.9 MiB/s wr, 518 op/s 2024-09-13T13:03:12.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 170535 9.56 MB/sec execute 317 sec latency 64.001 ms 2024-09-13T13:03:13.579 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:13 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 19/1620 objects degraded (1.173%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:13.649 INFO:teuthology.orchestra.run.smithi042.stdout: 5 170887 9.56 MB/sec execute 318 sec latency 53.137 ms 2024-09-13T13:03:13.834 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:13 smithi042 podman[70930]: 2024-09-13 13:03:13.663950105 +0000 UTC m=+1.114152310 container remove 1416d0a7f7c38b777b1c257d7e73f5b168d56f9bb64deacb7e2f86bbcc2cb66f (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-13T13:03:13.834 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:13 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T13:03:13.835 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:13 smithi042 systemd[1]: Stopped Ceph osd.1 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:03:13.835 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:13 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.1.service: Consumed 2min 56.475s CPU time. 2024-09-13T13:03:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:13 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 19/1620 objects degraded (1.173%), 2 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:14.085 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:14 smithi042 systemd[1]: Starting Ceph osd.1 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:03:14.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:14 smithi042 ceph-mon[63970]: pgmap v94: 97 pgs: 2 active+undersized, 13 stale+active+clean, 2 active+undersized+degraded, 80 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.2 MiB/s wr, 303 op/s; 19/1620 objects degraded (1.173%) 2024-09-13T13:03:14.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:14.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:14.593 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:14 smithi042 podman[71022]: 2024-09-13 13:03:14.255130744 +0000 UTC m=+0.025900519 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:03:14.593 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:14 smithi042 podman[71022]: 2024-09-13 13:03:14.460941599 +0000 UTC m=+0.231711353 container create 78907cf254fb86c34ac08126c0a37de2cef45c3f56f58475e3cc07593349c252 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-13T13:03:14.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 171198 9.55 MB/sec execute 319 sec latency 66.169 ms 2024-09-13T13:03:14.844 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:14 smithi042 podman[71022]: 2024-09-13 13:03:14.815534131 +0000 UTC m=+0.586303891 container init 78907cf254fb86c34ac08126c0a37de2cef45c3f56f58475e3cc07593349c252 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:03:14.844 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:14 smithi042 podman[71022]: 2024-09-13 13:03:14.829639186 +0000 UTC m=+0.600408955 container start 78907cf254fb86c34ac08126c0a37de2cef45c3f56f58475e3cc07593349c252 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T13:03:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:14 smithi046 ceph-mon[55972]: pgmap v94: 97 pgs: 2 active+undersized, 13 stale+active+clean, 2 active+undersized+degraded, 80 active+clean; 348 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.1 KiB/s rd, 3.2 MiB/s wr, 303 op/s; 19/1620 objects degraded (1.173%) 2024-09-13T13:03:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:15.114 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:15 smithi042 podman[71022]: 2024-09-13 13:03:15.008036862 +0000 UTC m=+0.778806620 container attach 78907cf254fb86c34ac08126c0a37de2cef45c3f56f58475e3cc07593349c252 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=) 2024-09-13T13:03:15.114 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:15 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:03:15.114 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:15 smithi042 bash[71022]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:03:15.114 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:15 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:03:15.114 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:15 smithi042 bash[71022]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:03:15.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 171409 9.54 MB/sec execute 320 sec latency 54.187 ms 2024-09-13T13:03:16.639 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:16 smithi042 ceph-mon[63970]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 525 op/s; 188/1683 objects degraded (11.171%) 2024-09-13T13:03:16.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 171841 9.55 MB/sec execute 321 sec latency 67.461 ms 2024-09-13T13:03:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:16 smithi046 ceph-mon[55972]: pgmap v95: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.9 MiB/s wr, 525 op/s; 188/1683 objects degraded (11.171%) 2024-09-13T13:03:17.645 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:17 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T13:03:17.645 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:17 smithi042 bash[71022]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T13:03:17.645 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:17 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T13:03:17.646 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:17 smithi042 bash[71022]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T13:03:17.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 172517 9.57 MB/sec execute 322 sec latency 36.833 ms 2024-09-13T13:03:18.149 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T13:03:18.149 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 bash[71022]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T13:03:18.149 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 bash[71022]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 bash[71022]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 bash[71022]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate[71042]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 bash[71022]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T13:03:18.150 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:18 smithi042 podman[71022]: 2024-09-13 13:03:18.063479901 +0000 UTC m=+3.834249660 container died 78907cf254fb86c34ac08126c0a37de2cef45c3f56f58475e3cc07593349c252 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T13:03:18.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 172944 9.57 MB/sec execute 323 sec latency 54.054 ms 2024-09-13T13:03:18.651 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:18 smithi042 ceph-mon[63970]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.4 MiB/s wr, 302 op/s; 186/1695 objects degraded (10.973%) 2024-09-13T13:03:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:18 smithi046 ceph-mon[55972]: pgmap v96: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.4 KiB/s rd, 3.4 MiB/s wr, 302 op/s; 186/1695 objects degraded (10.973%) 2024-09-13T13:03:19.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 173289 9.58 MB/sec execute 324 sec latency 61.636 ms 2024-09-13T13:03:19.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:19 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 183/1695 objects degraded (10.796%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:19.676 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:19 smithi042 podman[71231]: 2024-09-13 13:03:19.434878684 +0000 UTC m=+1.362928664 container remove 78907cf254fb86c34ac08126c0a37de2cef45c3f56f58475e3cc07593349c252 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T13:03:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:19 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 183/1695 objects degraded (10.796%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:19.927 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:19 smithi042 podman[71287]: 2024-09-13 13:03:19.602394163 +0000 UTC m=+0.017754694 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:03:19.927 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:19 smithi042 podman[71287]: 2024-09-13 13:03:19.744080465 +0000 UTC m=+0.159440985 container create d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T13:03:20.190 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:20 smithi042 podman[71287]: 2024-09-13 13:03:20.156827615 +0000 UTC m=+0.572188146 container init d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:03:20.190 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:20 smithi042 podman[71287]: 2024-09-13 13:03:20.166706615 +0000 UTC m=+0.582067137 container start d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-13T13:03:20.456 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:20 smithi042 ceph-osd[71304]: -- 172.21.15.42:0/2419581571 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ba9e1ba1e0 con 0x55ba9d355c00 2024-09-13T13:03:20.456 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:20 smithi042 bash[71287]: d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf 2024-09-13T13:03:20.456 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:20 smithi042 systemd[1]: Started Ceph osd.1 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:03:20.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 173558 9.56 MB/sec execute 325 sec latency 70.348 ms 2024-09-13T13:03:20.721 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:20 smithi042 ceph-mon[63970]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 409 op/s; 183/1695 objects degraded (10.796%) 2024-09-13T13:03:20.721 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:20.721 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:20.721 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:03:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:20 smithi046 ceph-mon[55972]: pgmap v97: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 409 op/s; 183/1695 objects degraded (10.796%) 2024-09-13T13:03:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:03:21.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 173811 9.57 MB/sec execute 326 sec latency 59.879 ms 2024-09-13T13:03:22.241 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:22 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:03:22.008+0000 7fe11f536640 -1 Falling back to public interface 2024-09-13T13:03:22.503 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:22 smithi042 ceph-mon[63970]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 372 op/s; 183/1695 objects degraded (10.796%) 2024-09-13T13:03:22.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 174134 9.57 MB/sec execute 327 sec latency 57.116 ms 2024-09-13T13:03:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:22 smithi046 ceph-mon[55972]: pgmap v98: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.2 MiB/s wr, 372 op/s; 183/1695 objects degraded (10.796%) 2024-09-13T13:03:23.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 174514 9.57 MB/sec execute 328 sec latency 86.175 ms 2024-09-13T13:03:24.551 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:24 smithi042 ceph-mon[63970]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 449 op/s; 187/1692 objects degraded (11.052%) 2024-09-13T13:03:24.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 174850 9.56 MB/sec execute 329 sec latency 52.692 ms 2024-09-13T13:03:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:24 smithi046 ceph-mon[55972]: pgmap v99: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 4.7 MiB/s wr, 449 op/s; 187/1692 objects degraded (11.052%) 2024-09-13T13:03:25.570 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:25 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 187/1692 objects degraded (11.052%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:25.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 175212 9.56 MB/sec execute 330 sec latency 58.004 ms 2024-09-13T13:03:25.876 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:25 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 187/1692 objects degraded (11.052%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:26.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:26 smithi042 ceph-mon[63970]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 556 op/s; 191/1710 objects degraded (11.170%) 2024-09-13T13:03:26.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 175614 9.56 MB/sec execute 331 sec latency 66.653 ms 2024-09-13T13:03:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:26 smithi046 ceph-mon[55972]: pgmap v100: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.9 MiB/s wr, 556 op/s; 191/1710 objects degraded (11.170%) 2024-09-13T13:03:27.650 INFO:teuthology.orchestra.run.smithi042.stdout: 5 175938 9.56 MB/sec execute 332 sec latency 93.736 ms 2024-09-13T13:03:28.597 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:28 smithi042 ceph-mon[63970]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 486 op/s; 190/1707 objects degraded (11.131%) 2024-09-13T13:03:28.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 176264 9.55 MB/sec execute 333 sec latency 49.809 ms 2024-09-13T13:03:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:28 smithi046 ceph-mon[55972]: pgmap v101: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 5.0 MiB/s wr, 486 op/s; 190/1707 objects degraded (11.131%) 2024-09-13T13:03:29.099 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:29 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:03:29.063+0000 7fe11f536640 -1 osd.1 0 read_superblock omap replica is missing. 2024-09-13T13:03:29.606 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:29.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 176570 9.55 MB/sec execute 334 sec latency 69.316 ms 2024-09-13T13:03:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:30.367 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:30 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:03:30.243+0000 7fe11f536640 -1 osd.1 87 log_to_monitors true 2024-09-13T13:03:30.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:30 smithi042 ceph-mon[63970]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 645 op/s; 179/1608 objects degraded (11.132%) 2024-09-13T13:03:30.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:30.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:30.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:30 smithi042 ceph-mon[63970]: from='osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T13:03:30.619 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:30 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 179/1608 objects degraded (11.132%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:30.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 176926 9.55 MB/sec execute 335 sec latency 78.993 ms 2024-09-13T13:03:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:30 smithi046 ceph-mon[55972]: pgmap v102: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 6.2 MiB/s wr, 645 op/s; 179/1608 objects degraded (11.132%) 2024-09-13T13:03:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:30 smithi046 ceph-mon[55972]: from='osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-13T13:03:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:30 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 179/1608 objects degraded (11.132%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:31.122 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:03:31.013+0000 7fe1172e0640 -1 osd.1 87 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:03:31.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 177276 9.56 MB/sec execute 336 sec latency 39.220 ms 2024-09-13T13:03:31.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:31.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:31.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-mon[63970]: from='osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T13:03:31.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-mon[63970]: osdmap e90: 8 total, 7 up, 8 in 2024-09-13T13:03:31.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-mon[63970]: from='osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:03:31.880 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:31 smithi042 ceph-mon[63970]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 599 op/s; 179/1608 objects degraded (11.132%) 2024-09-13T13:03:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:31 smithi046 ceph-mon[55972]: from='osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T13:03:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:31 smithi046 ceph-mon[55972]: osdmap e90: 8 total, 7 up, 8 in 2024-09-13T13:03:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:31 smithi046 ceph-mon[55972]: from='osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:03:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:31 smithi046 ceph-mon[55972]: pgmap v104: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 352 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 5.4 MiB/s wr, 599 op/s; 179/1608 objects degraded (11.132%) 2024-09-13T13:03:32.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 177424 9.54 MB/sec execute 337 sec latency 639.489 ms 2024-09-13T13:03:32.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:32 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:03:32.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:32 smithi042 ceph-mon[63970]: osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174] boot 2024-09-13T13:03:32.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:32 smithi042 ceph-mon[63970]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T13:03:32.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:32 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T13:03:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:32 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:03:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:32 smithi046 ceph-mon[55972]: osd.1 [v2:172.21.15.42:6810/2193459174,v1:172.21.15.42:6811/2193459174] boot 2024-09-13T13:03:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:32 smithi046 ceph-mon[55972]: osdmap e91: 8 total, 8 up, 8 in 2024-09-13T13:03:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:32 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-13T13:03:33.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 177532 9.52 MB/sec execute 338 sec latency 1639.531 ms 2024-09-13T13:03:34.044 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:33 smithi042 ceph-mon[63970]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T13:03:34.045 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:33 smithi042 ceph-mon[63970]: pgmap v107: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 387 op/s; 177/1617 objects degraded (10.946%) 2024-09-13T13:03:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:33 smithi046 ceph-mon[55972]: osdmap e92: 8 total, 8 up, 8 in 2024-09-13T13:03:34.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:33 smithi046 ceph-mon[55972]: pgmap v107: 97 pgs: 15 active+undersized, 24 active+undersized+degraded, 58 active+clean; 351 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 387 op/s; 177/1617 objects degraded (10.946%) 2024-09-13T13:03:34.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 177863 9.53 MB/sec execute 339 sec latency 2042.709 ms 2024-09-13T13:03:35.067 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:34 smithi042 ceph-mon[63970]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T13:03:35.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:34 smithi046 ceph-mon[55972]: osdmap e93: 8 total, 8 up, 8 in 2024-09-13T13:03:35.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 178315 9.53 MB/sec execute 340 sec latency 440.488 ms 2024-09-13T13:03:36.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:35 smithi042 ceph-mon[63970]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T13:03:36.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:35 smithi042 ceph-mon[63970]: pgmap v110: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized, 6 active+undersized+degraded, 74 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 447 op/s; 220/1602 objects degraded (13.733%); 956 KiB/s, 12 objects/s recovering 2024-09-13T13:03:36.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:35 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 177/1617 objects degraded (10.946%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:36.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:35 smithi042 ceph-mon[63970]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T13:03:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:35 smithi046 ceph-mon[55972]: osdmap e94: 8 total, 8 up, 8 in 2024-09-13T13:03:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:35 smithi046 ceph-mon[55972]: pgmap v110: 97 pgs: 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+undersized, 6 active+undersized+degraded, 74 active+clean; 354 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 5.2 MiB/s wr, 447 op/s; 220/1602 objects degraded (13.733%); 956 KiB/s, 12 objects/s recovering 2024-09-13T13:03:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:35 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 177/1617 objects degraded (10.946%), 24 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:35 smithi046 ceph-mon[55972]: osdmap e95: 8 total, 8 up, 8 in 2024-09-13T13:03:36.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 178576 9.52 MB/sec execute 341 sec latency 275.048 ms 2024-09-13T13:03:37.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 179006 9.53 MB/sec execute 342 sec latency 611.509 ms 2024-09-13T13:03:37.662 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T13:03:37.662 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.662 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:37.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:37.664 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:37.664 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:37 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T13:03:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: osdmap e96: 8 total, 8 up, 8 in 2024-09-13T13:03:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:03:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:03:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:03:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:37 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T13:03:38.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 179640 9.54 MB/sec execute 343 sec latency 312.466 ms 2024-09-13T13:03:38.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:38 smithi042 ceph-mon[63970]: pgmap v113: 97 pgs: 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 427 op/s; 204/1620 objects degraded (12.593%); 2.9 MiB/s, 71 keys/s, 12 objects/s recovering 2024-09-13T13:03:38.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:38 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T13:03:38.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:38 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T13:03:38.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:38 smithi042 ceph-mon[63970]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T13:03:38.769 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:38 smithi046 ceph-mon[55972]: pgmap v113: 97 pgs: 1 peering, 8 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 350 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.6 MiB/s wr, 427 op/s; 204/1620 objects degraded (12.593%); 2.9 MiB/s, 71 keys/s, 12 objects/s recovering 2024-09-13T13:03:38.769 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:38 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T13:03:38.769 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:38 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T13:03:38.769 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:38 smithi046 ceph-mon[55972]: osdmap e97: 8 total, 8 up, 8 in 2024-09-13T13:03:39.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 179961 9.53 MB/sec execute 344 sec latency 1032.392 ms 2024-09-13T13:03:39.680 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:39 smithi042 ceph-mon[63970]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T13:03:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:39 smithi046 ceph-mon[55972]: osdmap e98: 8 total, 8 up, 8 in 2024-09-13T13:03:40.505 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:40 smithi042 ceph-mon[63970]: pgmap v116: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 713 op/s; 112/1740 objects degraded (6.437%); 2.0 MiB/s, 71 keys/s, 23 objects/s recovering 2024-09-13T13:03:40.505 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:40 smithi042 ceph-mon[63970]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T13:03:40.505 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:40 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 112/1740 objects degraded (6.437%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:40.505 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:40 smithi042 ceph-mon[63970]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T13:03:40.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 180160 9.52 MB/sec execute 345 sec latency 778.362 ms 2024-09-13T13:03:40.757 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:03:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:40 smithi046 ceph-mon[55972]: pgmap v116: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.0 MiB/s wr, 713 op/s; 112/1740 objects degraded (6.437%); 2.0 MiB/s, 71 keys/s, 23 objects/s recovering 2024-09-13T13:03:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:40 smithi046 ceph-mon[55972]: osdmap e99: 8 total, 8 up, 8 in 2024-09-13T13:03:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:40 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 112/1740 objects degraded (6.437%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:40 smithi046 ceph-mon[55972]: osdmap e100: 8 total, 8 up, 8 in 2024-09-13T13:03:41.511 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:41 smithi042 ceph-mon[63970]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:41.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 180468 9.53 MB/sec execute 346 sec latency 692.788 ms 2024-09-13T13:03:41.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:41 smithi046 ceph-mon[55972]: from='client.44195 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:41.883 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (5m) 11s ago 17m 19.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (112s) 11s ago 17m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (106s) 102s ago 13m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (5m) 11s ago 14m 45.7M - dad864ee21e9 7144014c89b8 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (8m) 11s ago 8m 40.8M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (8m) 102s ago 8m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (4m) 11s ago 18m 516M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (3m) 102s ago 13m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (3m) 11s ago 18m 51.5M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (2m) 102s ago 13m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:03:41.884 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (8m) 11s ago 7m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (7m) 102s ago 7m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (4m) 11s ago 14m 9823k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (4m) 102s ago 13m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (71s) 11s ago 12m 233M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (22s) 11s ago 12m 12.1M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (11m) 11s ago 11m 390M 1937M 17.2.0 e1d6a67b021e 1c9ab19e7878 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (11m) 11s ago 11m 542M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (11m) 102s ago 11m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (10m) 102s ago 10m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (10m) 102s ago 10m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (9m) 102s ago 9m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:03:41.885 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (3m) 11s ago 13m 50.9M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:03:42.527 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:42 smithi042 ceph-mon[63970]: pgmap v119: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 865 op/s; 82/1722 objects degraded (4.762%); 0 B/s, 36 objects/s recovering 2024-09-13T13:03:42.527 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:42 smithi042 ceph-mon[63970]: from='client.44199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:42.528 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:42 smithi042 ceph-mon[63970]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T13:03:42.528 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:42 smithi042 ceph-mon[63970]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:03:42.648 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 6 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:03:42.649 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:03:42.652 INFO:teuthology.orchestra.run.smithi042.stdout: 5 180783 9.51 MB/sec execute 347 sec latency 62.705 ms 2024-09-13T13:03:42.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:42 smithi046 ceph-mon[55972]: pgmap v119: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.5 KiB/s rd, 7.2 MiB/s wr, 865 op/s; 82/1722 objects degraded (4.762%); 0 B/s, 36 objects/s recovering 2024-09-13T13:03:42.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:42 smithi046 ceph-mon[55972]: from='client.44199 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:42.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:42 smithi046 ceph-mon[55972]: osdmap e101: 8 total, 8 up, 8 in 2024-09-13T13:03:42.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:42 smithi046 ceph-mon[55972]: osdmap e102: 8 total, 8 up, 8 in 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:03:43.339 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:03:43.340 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:03:43.340 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T13:03:43.340 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:03:43.340 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:03:43.340 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:03:43.489 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:43 smithi042 ceph-mon[63970]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:43.489 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:43 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/4075307718' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:43.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 181210 9.53 MB/sec execute 348 sec latency 40.187 ms 2024-09-13T13:03:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:43 smithi046 ceph-mon[55972]: from='client.44203 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:43 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/4075307718' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:03:44.122 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN Degraded data redundancy: 52/1692 objects degraded (3.073%), 3 pgs degraded 2024-09-13T13:03:44.122 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 52/1692 objects degraded (3.073%), 3 pgs degraded 2024-09-13T13:03:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.16 is active+recovering+undersized+degraded+remapped, acting [5,7] 2024-09-13T13:03:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.18 is active+recovery_wait+undersized+degraded+remapped, acting [3,0] 2024-09-13T13:03:44.122 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1c is active+recovery_wait+undersized+degraded+remapped, acting [5,4] 2024-09-13T13:03:44.562 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:44 smithi042 ceph-mon[63970]: pgmap v122: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 236 op/s; 52/1692 objects degraded (3.073%); 129 KiB/s, 21 objects/s recovering 2024-09-13T13:03:44.562 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:44 smithi042 ceph-mon[63970]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:44.562 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:44.562 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:44.562 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:44 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2260251910' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:03:44.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 181869 9.56 MB/sec execute 349 sec latency 21.023 ms 2024-09-13T13:03:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:44 smithi046 ceph-mon[55972]: pgmap v122: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 2 peering, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 1.9 MiB/s wr, 236 op/s; 52/1692 objects degraded (3.073%); 129 KiB/s, 21 objects/s recovering 2024-09-13T13:03:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:44 smithi046 ceph-mon[55972]: from='client.44211 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:03:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:44 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2260251910' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:03:45.568 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:45 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 52/1692 objects degraded (3.073%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:45.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 182211 9.57 MB/sec execute 350 sec latency 65.694 ms 2024-09-13T13:03:45.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:45 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 52/1692 objects degraded (3.073%), 3 pgs degraded (PG_DEGRADED) 2024-09-13T13:03:46.584 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:46 smithi042 ceph-mon[63970]: pgmap v123: 97 pgs: 1 peering, 96 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 408 op/s; 88 KiB/s, 23 objects/s recovering 2024-09-13T13:03:46.584 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:46 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1692 objects degraded (3.073%), 3 pgs degraded) 2024-09-13T13:03:46.584 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:46 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:03:46.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 182494 9.56 MB/sec execute 351 sec latency 55.251 ms 2024-09-13T13:03:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:46 smithi046 ceph-mon[55972]: pgmap v123: 97 pgs: 1 peering, 96 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.3 MiB/s wr, 408 op/s; 88 KiB/s, 23 objects/s recovering 2024-09-13T13:03:46.902 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:46 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 52/1692 objects degraded (3.073%), 3 pgs degraded) 2024-09-13T13:03:46.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:46 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:03:47.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 182791 9.54 MB/sec execute 352 sec latency 59.223 ms 2024-09-13T13:03:48.631 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:48 smithi042 ceph-mon[63970]: pgmap v124: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.9 MiB/s wr, 452 op/s; 75 KiB/s, 20 objects/s recovering 2024-09-13T13:03:48.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 183416 9.56 MB/sec execute 353 sec latency 49.948 ms 2024-09-13T13:03:48.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:48 smithi046 ceph-mon[55972]: pgmap v124: 97 pgs: 97 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.9 MiB/s wr, 452 op/s; 75 KiB/s, 20 objects/s recovering 2024-09-13T13:03:49.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 184102 9.59 MB/sec execute 354 sec latency 39.129 ms 2024-09-13T13:03:50.657 INFO:teuthology.orchestra.run.smithi042.stdout: 5 184761 9.61 MB/sec execute 355 sec latency 37.466 ms 2024-09-13T13:03:50.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:50 smithi042 ceph-mon[63970]: pgmap v125: 97 pgs: 97 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 666 op/s; 64 KiB/s, 10 objects/s recovering 2024-09-13T13:03:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:50 smithi046 ceph-mon[55972]: pgmap v125: 97 pgs: 97 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 KiB/s rd, 7.0 MiB/s wr, 666 op/s; 64 KiB/s, 10 objects/s recovering 2024-09-13T13:03:51.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 185348 9.64 MB/sec execute 356 sec latency 32.707 ms 2024-09-13T13:03:52.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 185821 9.65 MB/sec execute 357 sec latency 59.599 ms 2024-09-13T13:03:52.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:52 smithi042 ceph-mon[63970]: pgmap v126: 97 pgs: 97 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 538 op/s; 0 B/s, 5 objects/s recovering 2024-09-13T13:03:52.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T13:03:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:52 smithi046 ceph-mon[55972]: pgmap v126: 97 pgs: 97 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 5.4 MiB/s wr, 538 op/s; 0 B/s, 5 objects/s recovering 2024-09-13T13:03:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-13T13:03:53.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 186177 9.64 MB/sec execute 358 sec latency 45.116 ms 2024-09-13T13:03:53.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:53 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T13:03:53.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:53 smithi042 ceph-mon[63970]: Upgrade: osd.2 is safe to restart 2024-09-13T13:03:53.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:53 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T13:03:53.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:53 smithi046 ceph-mon[55972]: Upgrade: osd.2 is safe to restart 2024-09-13T13:03:54.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 186448 9.63 MB/sec execute 359 sec latency 60.523 ms 2024-09-13T13:03:54.727 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:54 smithi042 ceph-mon[63970]: pgmap v127: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.6 MiB/s wr, 516 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:03:54.727 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:54 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:54 smithi046 ceph-mon[55972]: pgmap v127: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.6 MiB/s wr, 516 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:03:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:54 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:55.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 186725 9.62 MB/sec execute 360 sec latency 69.833 ms 2024-09-13T13:03:55.797 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:55 smithi042 ceph-mon[63970]: Upgrade: Updating osd.2 2024-09-13T13:03:55.797 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:55.797 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T13:03:55.797 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:03:55.798 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:55 smithi042 ceph-mon[63970]: Deploying daemon osd.2 on smithi042 2024-09-13T13:03:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:55 smithi046 ceph-mon[55972]: Upgrade: Updating osd.2 2024-09-13T13:03:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:55 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:55 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-13T13:03:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:55 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:03:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:55 smithi046 ceph-mon[55972]: Deploying daemon osd.2 on smithi042 2024-09-13T13:03:56.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 187011 9.61 MB/sec execute 361 sec latency 47.777 ms 2024-09-13T13:03:56.828 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:56 smithi042 ceph-mon[63970]: pgmap v128: 97 pgs: 97 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 743 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:03:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:56 smithi046 ceph-mon[55972]: pgmap v128: 97 pgs: 97 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 11 KiB/s rd, 7.7 MiB/s wr, 743 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:03:57.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 187299 9.61 MB/sec execute 362 sec latency 79.548 ms 2024-09-13T13:03:58.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 187645 9.62 MB/sec execute 363 sec latency 71.734 ms 2024-09-13T13:03:58.773 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:58 smithi042 ceph-mon[63970]: pgmap v129: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.5 MiB/s wr, 641 op/s 2024-09-13T13:03:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:58 smithi046 ceph-mon[55972]: pgmap v129: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 6.5 MiB/s wr, 641 op/s 2024-09-13T13:03:59.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 187943 9.60 MB/sec execute 364 sec latency 49.358 ms 2024-09-13T13:03:59.876 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:03:59 smithi042 systemd[1]: Stopping Ceph osd.2 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:03:59.877 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:03:59.877 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:03:59.877 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:03:59 smithi042 ceph-mon[63970]: pgmap v130: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 747 op/s 2024-09-13T13:04:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:03:59 smithi046 ceph-mon[55972]: pgmap v130: 97 pgs: 97 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 KiB/s rd, 7.1 MiB/s wr, 747 op/s 2024-09-13T13:04:00.393 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[41966]: 2024-09-13T13:04:00.139+0000 7fe9035b1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:04:00.394 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[41966]: 2024-09-13T13:04:00.139+0000 7fe9035b1700 -1 osd.2 102 *** Got signal Terminated *** 2024-09-13T13:04:00.394 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[41966]: 2024-09-13T13:04:00.139+0000 7fe9035b1700 -1 osd.2 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:04:00.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 188252 9.60 MB/sec execute 365 sec latency 54.094 ms 2024-09-13T13:04:00.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:00 smithi042 ceph-mon[63970]: osd.2 marked itself down and dead 2024-09-13T13:04:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:00 smithi046 ceph-mon[55972]: osd.2 marked itself down and dead 2024-09-13T13:04:01.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 188305 9.58 MB/sec execute 366 sec latency 841.248 ms 2024-09-13T13:04:01.742 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:01 smithi042 podman[73178]: 2024-09-13 13:04:01.469302214 +0000 UTC m=+1.473182383 container died 1c9ab19e7878b8e2d0069261306a570a17dca9277fd1a1d103efc1b77a1891c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, version=8, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.openshift.expose-services=, io.buildah.version=1.19.8, ceph=True, distribution-scope=public, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-09-13T13:04:02.087 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:01 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:04:02.087 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:01 smithi042 ceph-mon[63970]: osdmap e103: 8 total, 7 up, 8 in 2024-09-13T13:04:02.087 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:01 smithi042 ceph-mon[63970]: pgmap v132: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 588 op/s 2024-09-13T13:04:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:01 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:04:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:01 smithi046 ceph-mon[55972]: osdmap e103: 8 total, 7 up, 8 in 2024-09-13T13:04:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:01 smithi046 ceph-mon[55972]: pgmap v132: 97 pgs: 6 peering, 3 stale+active+clean, 88 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.3 KiB/s rd, 5.9 MiB/s wr, 588 op/s 2024-09-13T13:04:02.341 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:02 smithi042 podman[73178]: 2024-09-13 13:04:02.246999872 +0000 UTC m=+2.250880054 container cleanup 1c9ab19e7878b8e2d0069261306a570a17dca9277fd1a1d103efc1b77a1891c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, ceph=True, distribution-scope=public, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, io.openshift.expose-services=, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754) 2024-09-13T13:04:02.341 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:02 smithi042 bash[73178]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2 2024-09-13T13:04:02.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 188533 9.57 MB/sec execute 367 sec latency 1041.336 ms 2024-09-13T13:04:02.842 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:02 smithi042 ceph-mon[63970]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T13:04:02.842 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:02 smithi042 ceph-mon[63970]: osdmap e104: 8 total, 7 up, 8 in 2024-09-13T13:04:02.842 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:02 smithi042 podman[73194]: 2024-09-13 13:04:02.817502939 +0000 UTC m=+1.335726286 container remove 1c9ab19e7878b8e2d0069261306a570a17dca9277fd1a1d103efc1b77a1891c9 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_CLEAN=True, version=8, io.buildah.version=1.19.8) 2024-09-13T13:04:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:02 smithi046 ceph-mon[55972]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-13T13:04:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:02 smithi046 ceph-mon[55972]: osdmap e104: 8 total, 7 up, 8 in 2024-09-13T13:04:03.350 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:03 smithi042 podman[73276]: 2024-09-13 13:04:03.16636455 +0000 UTC m=+0.035765460 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:04:03.350 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:03 smithi042 podman[73276]: 2024-09-13 13:04:03.302144673 +0000 UTC m=+0.171545572 container create 15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:04:03.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 188973 9.59 MB/sec execute 368 sec latency 30.020 ms 2024-09-13T13:04:03.852 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:03 smithi042 podman[73276]: 2024-09-13 13:04:03.670554472 +0000 UTC m=+0.539955363 container init 15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:04:03.852 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:03 smithi042 podman[73276]: 2024-09-13 13:04:03.683185593 +0000 UTC m=+0.552586492 container start 15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:04:03.852 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:03 smithi042 podman[73276]: 2024-09-13 13:04:03.840614625 +0000 UTC m=+0.710015521 container attach 15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-13T13:04:03.853 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:03 smithi042 ceph-mon[63970]: pgmap v134: 97 pgs: 8 peering, 3 stale+active+clean, 86 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.9 MiB/s wr, 301 op/s 2024-09-13T13:04:04.104 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:04 smithi042 conmon[73289]: conmon 15a5cafde221eaf6a14c : Failed to open cgroups file: /sys/fs/cgroup/machine.slice/libpod-15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0.scope/container/memory.events 2024-09-13T13:04:04.104 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:04 smithi042 podman[73276]: 2024-09-13 13:04:04.080753078 +0000 UTC m=+0.950153986 container died 15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2) 2024-09-13T13:04:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:03 smithi046 ceph-mon[55972]: pgmap v134: 97 pgs: 8 peering, 3 stale+active+clean, 86 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 2.9 MiB/s wr, 301 op/s 2024-09-13T13:04:04.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 189478 9.60 MB/sec execute 369 sec latency 39.243 ms 2024-09-13T13:04:05.366 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 podman[73297]: 2024-09-13 13:04:05.239359461 +0000 UTC m=+1.150389032 container remove 15a5cafde221eaf6a14ccb74d4ba33ff428636c96abe133e25ed0247216bb2f0 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T13:04:05.366 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T13:04:05.366 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 systemd[1]: Stopped Ceph osd.2 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:04:05.366 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.2.service: Consumed 2min 15.044s CPU time. 2024-09-13T13:04:05.617 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:05 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 113/1680 objects degraded (6.726%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:05.653 INFO:teuthology.orchestra.run.smithi042.stdout: 5 189840 9.59 MB/sec execute 370 sec latency 75.905 ms 2024-09-13T13:04:05.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:05 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 113/1680 objects degraded (6.726%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:05.870 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 systemd[1]: Starting Ceph osd.2 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:04:06.121 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 podman[73391]: 2024-09-13 13:04:05.780702737 +0000 UTC m=+0.021931913 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:04:06.121 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:05 smithi042 podman[73391]: 2024-09-13 13:04:05.909062825 +0000 UTC m=+0.150291986 container create 0ae48d1f42a6b753c5dfa7117a7d468eddd904ed9e8d548609930c21601ba7c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-13T13:04:06.497 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:06 smithi042 ceph-mon[63970]: pgmap v135: 97 pgs: 3 active+undersized, 8 peering, 14 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 436 op/s; 113/1680 objects degraded (6.726%) 2024-09-13T13:04:06.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 190089 9.58 MB/sec execute 371 sec latency 60.627 ms 2024-09-13T13:04:06.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:06 smithi046 ceph-mon[55972]: pgmap v135: 97 pgs: 3 active+undersized, 8 peering, 14 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.9 KiB/s rd, 4.6 MiB/s wr, 436 op/s; 113/1680 objects degraded (6.726%) 2024-09-13T13:04:07.011 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 podman[73391]: 2024-09-13 13:04:06.766190967 +0000 UTC m=+1.007420136 container init 0ae48d1f42a6b753c5dfa7117a7d468eddd904ed9e8d548609930c21601ba7c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T13:04:07.011 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 podman[73391]: 2024-09-13 13:04:06.781068235 +0000 UTC m=+1.022297400 container start 0ae48d1f42a6b753c5dfa7117a7d468eddd904ed9e8d548609930c21601ba7c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T13:04:07.011 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:07.012 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 bash[73391]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:07.012 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 podman[73391]: 2024-09-13 13:04:06.950496672 +0000 UTC m=+1.191725841 container attach 0ae48d1f42a6b753c5dfa7117a7d468eddd904ed9e8d548609930c21601ba7c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-13T13:04:07.012 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:07.012 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:06 smithi042 bash[73391]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:07.514 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:07 smithi042 ceph-mon[63970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T13:04:07.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 190399 9.57 MB/sec execute 372 sec latency 50.945 ms 2024-09-13T13:04:07.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:07 smithi046 ceph-mon[55972]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T13:04:08.569 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:08 smithi042 ceph-mon[63970]: pgmap v136: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 359 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.9 MiB/s wr, 341 op/s; 161/1701 objects degraded (9.465%) 2024-09-13T13:04:08.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 190702 9.57 MB/sec execute 373 sec latency 58.259 ms 2024-09-13T13:04:08.768 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:08 smithi046 ceph-mon[55972]: pgmap v136: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 359 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.0 KiB/s rd, 3.9 MiB/s wr, 341 op/s; 161/1701 objects degraded (9.465%) 2024-09-13T13:04:09.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 191209 9.59 MB/sec execute 374 sec latency 75.646 ms 2024-09-13T13:04:10.087 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T13:04:10.087 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T13:04:10.087 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T13:04:10.087 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T13:04:10.590 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T13:04:10.590 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 bash[73391]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate[73406]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T13:04:10.591 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:10 smithi042 podman[73391]: 2024-09-13 13:04:10.47887062 +0000 UTC m=+4.720099782 container died 0ae48d1f42a6b753c5dfa7117a7d468eddd904ed9e8d548609930c21601ba7c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T13:04:10.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-mon[63970]: pgmap v137: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 555 op/s; 144/1599 objects degraded (9.006%) 2024-09-13T13:04:10.592 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:10 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 144/1599 objects degraded (9.006%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:10.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 191642 9.59 MB/sec execute 375 sec latency 43.632 ms 2024-09-13T13:04:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:10 smithi046 ceph-mon[55972]: pgmap v137: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 5.5 MiB/s wr, 555 op/s; 144/1599 objects degraded (9.006%) 2024-09-13T13:04:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:10 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 144/1599 objects degraded (9.006%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:11.607 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:11 smithi042 podman[73562]: 2024-09-13 13:04:11.442911381 +0000 UTC m=+0.951478022 container remove 0ae48d1f42a6b753c5dfa7117a7d468eddd904ed9e8d548609930c21601ba7c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-activate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T13:04:11.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 192078 9.60 MB/sec execute 376 sec latency 83.356 ms 2024-09-13T13:04:11.859 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:11 smithi042 podman[73616]: 2024-09-13 13:04:11.613908295 +0000 UTC m=+0.024147495 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:04:11.859 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:11 smithi042 podman[73616]: 2024-09-13 13:04:11.752496216 +0000 UTC m=+0.162735394 container create 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:04:12.110 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:12 smithi042 podman[73616]: 2024-09-13 13:04:12.055415844 +0000 UTC m=+0.465655031 container init 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T13:04:12.110 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:12 smithi042 podman[73616]: 2024-09-13 13:04:12.06737219 +0000 UTC m=+0.477611370 container start 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T13:04:12.110 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:12 smithi042 ceph-osd[73634]: -- 172.21.15.42:0/654196110 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a8121fe1e0 con 0x55a811399c00 2024-09-13T13:04:12.366 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:12 smithi042 bash[73616]: 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b 2024-09-13T13:04:12.366 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:12 smithi042 systemd[1]: Started Ceph osd.2 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:04:12.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:12 smithi042 ceph-mon[63970]: pgmap v138: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 144/1590 objects degraded (9.057%) 2024-09-13T13:04:12.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:12.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:12.618 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:04:12.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 192327 9.60 MB/sec execute 377 sec latency 100.413 ms 2024-09-13T13:04:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:12 smithi046 ceph-mon[55972]: pgmap v138: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 350 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.7 MiB/s wr, 472 op/s; 144/1590 objects degraded (9.057%) 2024-09-13T13:04:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:04:13.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 192640 9.60 MB/sec execute 378 sec latency 53.493 ms 2024-09-13T13:04:13.884 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:13 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:04:13.637+0000 7f87a9034640 -1 Falling back to public interface 2024-09-13T13:04:14.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 192874 9.59 MB/sec execute 379 sec latency 132.565 ms 2024-09-13T13:04:14.656 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:04:14.659 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:14 smithi042 ceph-mon[63970]: pgmap v139: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 420 op/s; 146/1638 objects degraded (8.913%) 2024-09-13T13:04:14.659 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:14.659 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:14.659 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:14 smithi046 ceph-mon[55972]: pgmap v139: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 351 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.3 MiB/s wr, 420 op/s; 146/1638 objects degraded (8.913%) 2024-09-13T13:04:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:15.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 193168 9.58 MB/sec execute 380 sec latency 65.140 ms 2024-09-13T13:04:15.686 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:15 smithi042 ceph-mon[63970]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:15.686 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:15 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 146/1638 objects degraded (8.913%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:15.783 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:04:15.783 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (5m) 45s ago 17m 19.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:04:15.784 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (2m) 45s ago 17m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:04:15.784 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (2m) 2m ago 14m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:04:15.784 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (5m) 45s ago 14m 45.7M - dad864ee21e9 7144014c89b8 2024-09-13T13:04:15.784 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (8m) 45s ago 8m 40.8M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:04:15.785 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (8m) 2m ago 8m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:04:15.785 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (4m) 45s ago 19m 516M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:04:15.785 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:04:15.785 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (3m) 45s ago 19m 51.5M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:04:15.785 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (3m) 2m ago 14m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:04:15.786 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (8m) 45s ago 8m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:04:15.786 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (8m) 2m ago 8m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:04:15.786 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (5m) 45s ago 14m 9823k - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:04:15.786 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (5m) 2m ago 14m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:04:15.786 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (105s) 45s ago 13m 233M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:04:15.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (56s) 45s ago 12m 12.1M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:04:15.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 starting - - - 1937M 2024-09-13T13:04:15.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (12m) 45s ago 12m 542M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:04:15.787 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (11m) 2m ago 11m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:04:15.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (11m) 2m ago 11m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:04:15.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (10m) 2m ago 10m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:04:15.788 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (10m) 2m ago 10m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:04:15.788 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (4m) 45s ago 13m 50.9M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:04:15.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:15 smithi046 ceph-mon[55972]: from='client.34280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:15.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:15 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 146/1638 objects degraded (8.913%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:16.437 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:04:16.438 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:04:16.438 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:04:16.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:16.438 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:04:16.438 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:04:16.438 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:04:16.439 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:16.440 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:04:16.440 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T13:04:16.440 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 6 2024-09-13T13:04:16.440 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:04:16.440 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:04:16.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 193420 9.57 MB/sec execute 381 sec latency 48.616 ms 2024-09-13T13:04:16.700 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:16 smithi042 ceph-mon[63970]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:16.701 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:16 smithi042 ceph-mon[63970]: pgmap v140: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 527 op/s; 156/1707 objects degraded (9.139%) 2024-09-13T13:04:16.701 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:16 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/295505092' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:16 smithi046 ceph-mon[55972]: from='client.34284 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:16 smithi046 ceph-mon[55972]: pgmap v140: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.7 KiB/s rd, 5.7 MiB/s wr, 527 op/s; 156/1707 objects degraded (9.139%) 2024-09-13T13:04:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:16 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/295505092' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:04:17.088 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:04:17.089 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:04:17.089 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "8/23 daemons upgraded", 2024-09-13T13:04:17.089 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:04:17.089 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:04:17.089 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:04:17.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 193714 9.56 MB/sec execute 382 sec latency 74.576 ms 2024-09-13T13:04:17.725 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:17 smithi042 ceph-mon[63970]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:17.791 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 158/1674 objects degraded (9.438%), 18 pgs degraded 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: osd.2 (root=default,host=smithi042) is down 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 158/1674 objects degraded (9.438%), 18 pgs degraded 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1 is active+undersized+degraded, acting [3,0] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.7 is active+undersized+degraded, acting [6,4] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.c is active+undersized+degraded, acting [5,0] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.e is active+undersized+degraded, acting [3,7] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.10 is active+undersized+degraded, acting [1,0] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.16 is active+undersized+degraded, acting [5,6] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.17 is active+undersized+degraded, acting [6,5] 2024-09-13T13:04:17.792 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1c is active+undersized+degraded, acting [7,5] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1f is active+undersized+degraded, acting [0,5] 2024-09-13T13:04:17.793 INFO:teuthology.orchestra.run.smithi042.stdout: pg 4.19 is active+undersized+degraded, acting [3,0] 2024-09-13T13:04:17.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:17 smithi046 ceph-mon[55972]: from='client.44223 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:18.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 194015 9.56 MB/sec execute 383 sec latency 76.282 ms 2024-09-13T13:04:18.762 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:18 smithi042 ceph-mon[63970]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:18.762 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:18 smithi042 ceph-mon[63970]: pgmap v141: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 549 op/s; 158/1674 objects degraded (9.438%) 2024-09-13T13:04:18.762 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:18 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/3274139270' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:04:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:18 smithi046 ceph-mon[55972]: from='client.34296 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:18 smithi046 ceph-mon[55972]: pgmap v141: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 360 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 5.2 MiB/s wr, 549 op/s; 158/1674 objects degraded (9.438%) 2024-09-13T13:04:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:18 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/3274139270' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:04:19.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 194287 9.54 MB/sec execute 384 sec latency 66.060 ms 2024-09-13T13:04:19.802 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:19 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:04:19.533+0000 7f87a9034640 -1 osd.2 0 read_superblock omap replica is missing. 2024-09-13T13:04:20.577 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:20 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:04:20.317+0000 7f87a9034640 -1 osd.2 102 log_to_monitors true 2024-09-13T13:04:20.577 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:20 smithi042 ceph-mon[63970]: pgmap v142: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.3 MiB/s wr, 505 op/s; 155/1629 objects degraded (9.515%) 2024-09-13T13:04:20.578 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:20 smithi042 ceph-mon[63970]: from='osd.2 [v2:172.21.15.42:6818/1541292689,v1:172.21.15.42:6819/1541292689]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T13:04:20.578 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:20 smithi042 ceph-mon[63970]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T13:04:20.578 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:20 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 155/1629 objects degraded (9.515%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:20.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 194563 9.55 MB/sec execute 385 sec latency 86.943 ms 2024-09-13T13:04:20.832 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:04:20 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:04:20.621+0000 7f87a0dde640 -1 osd.2 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:04:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:20 smithi046 ceph-mon[55972]: pgmap v142: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 9.9 KiB/s rd, 5.3 MiB/s wr, 505 op/s; 155/1629 objects degraded (9.515%) 2024-09-13T13:04:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:20 smithi046 ceph-mon[55972]: from='osd.2 [v2:172.21.15.42:6818/1541292689,v1:172.21.15.42:6819/1541292689]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T13:04:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:20 smithi046 ceph-mon[55972]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-13T13:04:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:20 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 155/1629 objects degraded (9.515%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:21.640 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:21 smithi042 ceph-mon[63970]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T13:04:21.640 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:21 smithi042 ceph-mon[63970]: osdmap e105: 8 total, 7 up, 8 in 2024-09-13T13:04:21.640 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:21 smithi042 ceph-mon[63970]: from='osd.2 [v2:172.21.15.42:6818/1541292689,v1:172.21.15.42:6819/1541292689]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:04:21.640 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:21 smithi042 ceph-mon[63970]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:04:21.640 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:21.641 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:21.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 194779 9.53 MB/sec execute 386 sec latency 111.635 ms 2024-09-13T13:04:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:21 smithi046 ceph-mon[55972]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T13:04:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:21 smithi046 ceph-mon[55972]: osdmap e105: 8 total, 7 up, 8 in 2024-09-13T13:04:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:21 smithi046 ceph-mon[55972]: from='osd.2 [v2:172.21.15.42:6818/1541292689,v1:172.21.15.42:6819/1541292689]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:04:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:21 smithi046 ceph-mon[55972]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:04:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:21.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:22.656 INFO:teuthology.orchestra.run.smithi042.stdout: 5 195018 9.53 MB/sec execute 387 sec latency 1102.683 ms 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: pgmap v144: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 408 op/s; 149/1611 objects degraded (9.249%) 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: osd.2 [v2:172.21.15.42:6818/1541292689,v1:172.21.15.42:6819/1541292689] boot 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:22.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: pgmap v144: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 4.7 MiB/s wr, 408 op/s; 149/1611 objects degraded (9.249%) 2024-09-13T13:04:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:04:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: osd.2 [v2:172.21.15.42:6818/1541292689,v1:172.21.15.42:6819/1541292689] boot 2024-09-13T13:04:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: osdmap e106: 8 total, 8 up, 8 in 2024-09-13T13:04:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-13T13:04:22.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:22.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:23.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 195174 9.51 MB/sec execute 388 sec latency 2102.713 ms 2024-09-13T13:04:23.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:23 smithi046 ceph-mon[55972]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T13:04:23.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:23 smithi046 ceph-mon[55972]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T13:04:23.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:23 smithi042 ceph-mon[63970]: osdmap e107: 8 total, 8 up, 8 in 2024-09-13T13:04:23.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:23 smithi042 ceph-mon[63970]: osdmap e108: 8 total, 8 up, 8 in 2024-09-13T13:04:24.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 195302 9.49 MB/sec execute 389 sec latency 2720.299 ms 2024-09-13T13:04:24.704 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:24 smithi042 ceph-mon[63970]: pgmap v147: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.0 MiB/s wr, 268 op/s; 148/1590 objects degraded (9.308%) 2024-09-13T13:04:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:24 smithi046 ceph-mon[55972]: pgmap v147: 97 pgs: 7 active+undersized, 18 active+undersized+degraded, 72 active+clean; 351 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.3 KiB/s rd, 3.0 MiB/s wr, 268 op/s; 148/1590 objects degraded (9.308%) 2024-09-13T13:04:25.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 195589 9.49 MB/sec execute 390 sec latency 424.875 ms 2024-09-13T13:04:25.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:25 smithi042 ceph-mon[63970]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T13:04:25.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:25 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 148/1590 objects degraded (9.308%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:25.860 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:25 smithi042 ceph-mon[63970]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T13:04:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:25 smithi046 ceph-mon[55972]: osdmap e109: 8 total, 8 up, 8 in 2024-09-13T13:04:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:25 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 148/1590 objects degraded (9.308%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:25 smithi046 ceph-mon[55972]: osdmap e110: 8 total, 8 up, 8 in 2024-09-13T13:04:26.625 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:26 smithi042 ceph-mon[63970]: pgmap v150: 97 pgs: 1 active+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 4 active+undersized, 7 active+undersized+degraded, 81 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 524 op/s; 158/1611 objects degraded (9.808%); 0 B/s, 5 objects/s recovering 2024-09-13T13:04:26.626 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:26 smithi042 ceph-mon[63970]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T13:04:26.654 INFO:teuthology.orchestra.run.smithi042.stdout: 5 195791 9.49 MB/sec execute 391 sec latency 414.928 ms 2024-09-13T13:04:27.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:26 smithi046 ceph-mon[55972]: pgmap v150: 97 pgs: 1 active+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 4 active+undersized, 7 active+undersized+degraded, 81 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 4.4 MiB/s wr, 524 op/s; 158/1611 objects degraded (9.808%); 0 B/s, 5 objects/s recovering 2024-09-13T13:04:27.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:26 smithi046 ceph-mon[55972]: osdmap e111: 8 total, 8 up, 8 in 2024-09-13T13:04:27.659 INFO:teuthology.orchestra.run.smithi042.stdout: 5 196173 9.48 MB/sec execute 392 sec latency 813.649 ms 2024-09-13T13:04:28.659 INFO:teuthology.orchestra.run.smithi042.stdout: 5 196307 9.46 MB/sec execute 393 sec latency 743.906 ms 2024-09-13T13:04:28.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: pgmap v153: 97 pgs: 2 peering, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 89 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.3 MiB/s wr, 348 op/s; 121/1623 objects degraded (7.455%); 403 KiB/s, 15 objects/s recovering 2024-09-13T13:04:28.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:28.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T13:04:28.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:28.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:04:28.675 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:28.676 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:28 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: pgmap v153: 97 pgs: 2 peering, 1 active+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 89 active+clean; 352 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 3.3 MiB/s wr, 348 op/s; 121/1623 objects degraded (7.455%); 403 KiB/s, 15 objects/s recovering 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: osdmap e112: 8 total, 8 up, 8 in 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:29.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:29.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:04:29.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:29.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:29.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:29.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:28 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T13:04:29.659 INFO:teuthology.orchestra.run.smithi042.stdout: 5 196587 9.47 MB/sec execute 394 sec latency 77.378 ms 2024-09-13T13:04:29.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:29 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T13:04:29.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:29 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T13:04:29.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:29 smithi042 ceph-mon[63970]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T13:04:29.694 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:29.695 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:29 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T13:04:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:29 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-09-13T13:04:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:29 smithi046 ceph-mon[55972]: osdmap e113: 8 total, 8 up, 8 in 2024-09-13T13:04:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:30.659 INFO:teuthology.orchestra.run.smithi042.stdout: 5 196805 9.46 MB/sec execute 395 sec latency 36.807 ms 2024-09-13T13:04:30.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:30 smithi042 ceph-mon[63970]: pgmap v156: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 469 op/s; 27/1653 objects degraded (1.633%); 1.4 MiB/s, 33 objects/s recovering 2024-09-13T13:04:30.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:30 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 27/1653 objects degraded (1.633%), 1 pg degraded (PG_DEGRADED) 2024-09-13T13:04:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:30 smithi046 ceph-mon[55972]: pgmap v156: 97 pgs: 2 peering, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 355 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 469 op/s; 27/1653 objects degraded (1.633%); 1.4 MiB/s, 33 objects/s recovering 2024-09-13T13:04:31.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:30 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 27/1653 objects degraded (1.633%), 1 pg degraded (PG_DEGRADED) 2024-09-13T13:04:31.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 197142 9.47 MB/sec execute 396 sec latency 57.459 ms 2024-09-13T13:04:31.786 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:31 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1653 objects degraded (1.633%), 1 pg degraded) 2024-09-13T13:04:31.786 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:31 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:04:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:31 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/1653 objects degraded (1.633%), 1 pg degraded) 2024-09-13T13:04:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:31 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:04:32.659 INFO:teuthology.orchestra.run.smithi042.stdout: 5 197463 9.45 MB/sec execute 397 sec latency 72.041 ms 2024-09-13T13:04:32.824 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:32 smithi042 ceph-mon[63970]: pgmap v157: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.3 MiB/s wr, 341 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-09-13T13:04:33.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:32 smithi046 ceph-mon[55972]: pgmap v157: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.3 MiB/s wr, 341 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-09-13T13:04:33.659 INFO:teuthology.orchestra.run.smithi042.stdout: 5 197713 9.45 MB/sec execute 398 sec latency 76.334 ms 2024-09-13T13:04:34.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 197855 9.43 MB/sec execute 399 sec latency 71.579 ms 2024-09-13T13:04:34.901 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:34 smithi042 ceph-mon[63970]: pgmap v158: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.9 MiB/s wr, 288 op/s; 1.2 MiB/s, 18 objects/s recovering 2024-09-13T13:04:35.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:34 smithi046 ceph-mon[55972]: pgmap v158: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.9 MiB/s wr, 288 op/s; 1.2 MiB/s, 18 objects/s recovering 2024-09-13T13:04:35.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 198207 9.44 MB/sec execute 400 sec latency 59.008 ms 2024-09-13T13:04:36.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 198443 9.43 MB/sec execute 401 sec latency 90.115 ms 2024-09-13T13:04:36.711 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:36 smithi042 ceph-mon[63970]: pgmap v159: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 460 op/s; 1.0 MiB/s, 15 objects/s recovering 2024-09-13T13:04:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:36 smithi046 ceph-mon[55972]: pgmap v159: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.4 MiB/s wr, 460 op/s; 1.0 MiB/s, 15 objects/s recovering 2024-09-13T13:04:37.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 198699 9.42 MB/sec execute 402 sec latency 91.054 ms 2024-09-13T13:04:38.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 198950 9.42 MB/sec execute 403 sec latency 57.196 ms 2024-09-13T13:04:38.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:38 smithi042 ceph-mon[63970]: pgmap v160: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 850 KiB/s, 12 objects/s recovering 2024-09-13T13:04:39.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:38 smithi046 ceph-mon[55972]: pgmap v160: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.4 KiB/s rd, 4.0 MiB/s wr, 404 op/s; 850 KiB/s, 12 objects/s recovering 2024-09-13T13:04:39.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 199093 9.41 MB/sec execute 404 sec latency 85.201 ms 2024-09-13T13:04:40.119 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:39 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:40.119 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:39 smithi042 ceph-mon[63970]: pgmap v161: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.2 MiB/s wr, 523 op/s; 771 KiB/s, 11 objects/s recovering 2024-09-13T13:04:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:39 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:40.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:39 smithi046 ceph-mon[55972]: pgmap v161: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 5.2 MiB/s wr, 523 op/s; 771 KiB/s, 11 objects/s recovering 2024-09-13T13:04:40.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 199398 9.40 MB/sec execute 405 sec latency 64.146 ms 2024-09-13T13:04:41.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 199691 9.39 MB/sec execute 406 sec latency 66.517 ms 2024-09-13T13:04:42.449 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:42 smithi042 ceph-mon[63970]: pgmap v162: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.3 MiB/s wr, 328 op/s; 344 KiB/s, 2 objects/s recovering 2024-09-13T13:04:42.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:42 smithi046 ceph-mon[55972]: pgmap v162: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.4 KiB/s rd, 3.3 MiB/s wr, 328 op/s; 344 KiB/s, 2 objects/s recovering 2024-09-13T13:04:42.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 200131 9.40 MB/sec execute 407 sec latency 36.731 ms 2024-09-13T13:04:43.508 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T13:04:43.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-13T13:04:43.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 200342 9.39 MB/sec execute 408 sec latency 59.272 ms 2024-09-13T13:04:44.532 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:44 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T13:04:44.532 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:44 smithi042 ceph-mon[63970]: Upgrade: osd.3 is safe to restart 2024-09-13T13:04:44.532 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:44 smithi042 ceph-mon[63970]: pgmap v163: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 331 op/s 2024-09-13T13:04:44.532 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:44.532 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:44 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T13:04:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:44 smithi046 ceph-mon[55972]: Upgrade: osd.3 is safe to restart 2024-09-13T13:04:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:44 smithi046 ceph-mon[55972]: pgmap v163: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 3.5 MiB/s wr, 331 op/s 2024-09-13T13:04:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:44.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:44.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 200673 9.40 MB/sec execute 409 sec latency 134.917 ms 2024-09-13T13:04:45.660 INFO:teuthology.orchestra.run.smithi042.stdout: 5 200975 9.39 MB/sec execute 410 sec latency 92.464 ms 2024-09-13T13:04:46.582 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:46 smithi042 ceph-mon[63970]: pgmap v164: 97 pgs: 97 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.7 MiB/s wr, 485 op/s 2024-09-13T13:04:46.582 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:46.582 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T13:04:46.582 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:46 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:04:46.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:46 smithi046 ceph-mon[55972]: pgmap v164: 97 pgs: 97 active+clean; 354 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.7 MiB/s wr, 485 op/s 2024-09-13T13:04:46.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:46.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-13T13:04:46.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:46 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:04:46.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 201300 9.39 MB/sec execute 411 sec latency 86.065 ms 2024-09-13T13:04:47.435 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:47 smithi042 ceph-mon[63970]: Upgrade: Updating osd.3 2024-09-13T13:04:47.435 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:47 smithi042 ceph-mon[63970]: Deploying daemon osd.3 on smithi042 2024-09-13T13:04:47.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 201614 9.38 MB/sec execute 412 sec latency 70.181 ms 2024-09-13T13:04:47.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:47 smithi046 ceph-mon[55972]: Upgrade: Updating osd.3 2024-09-13T13:04:47.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:47 smithi046 ceph-mon[55972]: Deploying daemon osd.3 on smithi042 2024-09-13T13:04:48.333 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:04:48.455 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:48 smithi042 ceph-mon[63970]: pgmap v165: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 361 op/s 2024-09-13T13:04:48.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 201919 9.38 MB/sec execute 413 sec latency 105.163 ms 2024-09-13T13:04:48.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:48 smithi046 ceph-mon[55972]: pgmap v165: 97 pgs: 97 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.5 KiB/s rd, 3.9 MiB/s wr, 361 op/s 2024-09-13T13:04:49.306 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:04:49.306 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (6m) 28s ago 18m 19.7M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:04:49.306 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (2m) 28s ago 18m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:04:49.306 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (2m) 2m ago 15m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:04:49.306 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (6m) 28s ago 15m 45.5M - dad864ee21e9 7144014c89b8 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (9m) 28s ago 9m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (9m) 2m ago 9m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (5m) 28s ago 19m 518M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (4m) 2m ago 14m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (4m) 28s ago 19m 53.7M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (3m) 2m ago 14m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (9m) 28s ago 9m 273M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (9m) 2m ago 9m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (6m) 28s ago 15m 9.78M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (5m) 2m ago 14m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (2m) 28s ago 14m 291M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (89s) 28s ago 13m 226M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:04:49.307 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (37s) 28s ago 13m 12.2M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:04:49.308 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (12m) 28s ago 12m 605M 1937M 17.2.0 e1d6a67b021e 7c9faf037aa3 2024-09-13T13:04:49.308 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (12m) 2m ago 12m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:04:49.308 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (11m) 2m ago 11m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:04:49.308 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (11m) 2m ago 11m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:04:49.308 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (10m) 2m ago 10m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:04:49.308 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (4m) 28s ago 14m 53.2M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:04:49.487 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:49 smithi042 ceph-mon[63970]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:49.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 202218 9.37 MB/sec execute 414 sec latency 80.028 ms 2024-09-13T13:04:49.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:49 smithi046 ceph-mon[55972]: from='client.34306 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:49.959 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:04:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:04:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:04:49.959 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 3 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:04:49.960 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-09-13T13:04:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 7 2024-09-13T13:04:49.961 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:04:49.961 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:04:50.450 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:04:50.450 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:04:50.450 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "9/23 daemons upgraded", 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:04:50.451 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:04:50.536 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:50 smithi042 ceph-mon[63970]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:50.537 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:50 smithi042 ceph-mon[63970]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:50.537 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:50 smithi042 ceph-mon[63970]: pgmap v166: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-09-13T13:04:50.537 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:50 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2506584816' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:50.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 202548 9.38 MB/sec execute 415 sec latency 67.432 ms 2024-09-13T13:04:50.827 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:50 smithi042 systemd[1]: Stopping Ceph osd.3 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:04:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:50 smithi046 ceph-mon[55972]: from='client.34310 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:50 smithi046 ceph-mon[55972]: from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:50 smithi046 ceph-mon[55972]: pgmap v166: 97 pgs: 97 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.0 MiB/s wr, 514 op/s 2024-09-13T13:04:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:50 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2506584816' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:04:51.053 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:04:51.109 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:51 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[45207]: 2024-09-13T13:04:51.079+0000 7f12066e5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:04:51.113 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:51 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[45207]: 2024-09-13T13:04:51.079+0000 7f12066e5700 -1 osd.3 113 *** Got signal Terminated *** 2024-09-13T13:04:51.113 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:51 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[45207]: 2024-09-13T13:04:51.079+0000 7f12066e5700 -1 osd.3 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:04:51.617 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:51 smithi042 ceph-mon[63970]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:51.617 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:51 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2534291475' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:04:51.617 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:51 smithi042 ceph-mon[63970]: osd.3 marked itself down and dead 2024-09-13T13:04:51.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 202769 9.37 MB/sec execute 416 sec latency 181.227 ms 2024-09-13T13:04:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:51 smithi046 ceph-mon[55972]: from='client.34322 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:04:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:51 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2534291475' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:04:51.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:51 smithi046 ceph-mon[55972]: osd.3 marked itself down and dead 2024-09-13T13:04:52.120 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:52 smithi042 podman[75655]: 2024-09-13 13:04:52.063863979 +0000 UTC m=+1.126605127 container died 7c9faf037aa3451422641593476faf29584aa2fef54d5283337df75e7f90ee1d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , ceph=True, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, name=centos-stream, distribution-scope=public, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, release=754, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, vendor=Red Hat, Inc.) 2024-09-13T13:04:52.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 202885 9.36 MB/sec execute 417 sec latency 1008.060 ms 2024-09-13T13:04:52.698 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:52 smithi042 ceph-mon[63970]: pgmap v167: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 381 op/s 2024-09-13T13:04:52.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:52 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:04:52.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:52 smithi042 ceph-mon[63970]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T13:04:52.699 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:52 smithi042 podman[75655]: 2024-09-13 13:04:52.695475357 +0000 UTC m=+1.758216498 container cleanup 7c9faf037aa3451422641593476faf29584aa2fef54d5283337df75e7f90ee1d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, ceph=True, io.buildah.version=1.19.8, release=754, architecture=x86_64, io.openshift.tags=base centos centos-stream, GIT_CLEAN=True, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, vendor=Red Hat, Inc., vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux ) 2024-09-13T13:04:52.699 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:52 smithi042 bash[75655]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3 2024-09-13T13:04:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:52 smithi046 ceph-mon[55972]: pgmap v167: 97 pgs: 97 active+clean; 353 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 381 op/s 2024-09-13T13:04:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:52 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:04:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:52 smithi046 ceph-mon[55972]: osdmap e114: 8 total, 7 up, 8 in 2024-09-13T13:04:53.214 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:53 smithi042 podman[75672]: 2024-09-13 13:04:53.057386118 +0000 UTC m=+0.996612948 container remove 7c9faf037aa3451422641593476faf29584aa2fef54d5283337df75e7f90ee1d (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, CEPH_POINT_RELEASE=-17.2.0, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.component=centos-stream-container, io.openshift.expose-services=, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., distribution-scope=public, GIT_CLEAN=True, vcs-type=git, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-09-13T13:04:53.472 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:53 smithi042 ceph-mon[63970]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T13:04:53.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 203116 9.36 MB/sec execute 418 sec latency 47.562 ms 2024-09-13T13:04:53.725 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:53 smithi042 podman[75755]: 2024-09-13 13:04:53.465679215 +0000 UTC m=+0.019983609 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:04:53.725 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:53 smithi042 podman[75755]: 2024-09-13 13:04:53.650853631 +0000 UTC m=+0.205158015 container create 4b98d2c08f101faa507064fb6cd8cb4c7342b2d6ece5f6a13f672074a7ee411e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:04:53.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:53 smithi046 ceph-mon[55972]: osdmap e115: 8 total, 7 up, 8 in 2024-09-13T13:04:54.481 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:54 smithi042 ceph-mon[63970]: pgmap v170: 97 pgs: 16 stale+active+clean, 81 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-09-13T13:04:54.482 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:54 smithi042 podman[75755]: 2024-09-13 13:04:54.330412875 +0000 UTC m=+0.884717302 container init 4b98d2c08f101faa507064fb6cd8cb4c7342b2d6ece5f6a13f672074a7ee411e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T13:04:54.482 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:54 smithi042 podman[75755]: 2024-09-13 13:04:54.338141474 +0000 UTC m=+0.892445863 container start 4b98d2c08f101faa507064fb6cd8cb4c7342b2d6ece5f6a13f672074a7ee411e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0) 2024-09-13T13:04:54.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 203362 9.34 MB/sec execute 419 sec latency 66.205 ms 2024-09-13T13:04:54.735 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:54 smithi042 podman[75755]: 2024-09-13 13:04:54.505913282 +0000 UTC m=+1.060217676 container attach 4b98d2c08f101faa507064fb6cd8cb4c7342b2d6ece5f6a13f672074a7ee411e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-13T13:04:54.735 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:54 smithi042 podman[75755]: 2024-09-13 13:04:54.652826068 +0000 UTC m=+1.207130461 container died 4b98d2c08f101faa507064fb6cd8cb4c7342b2d6ece5f6a13f672074a7ee411e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:04:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:54 smithi046 ceph-mon[55972]: pgmap v170: 97 pgs: 16 stale+active+clean, 81 active+clean; 352 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.4 MiB/s wr, 318 op/s 2024-09-13T13:04:55.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 203676 9.33 MB/sec execute 420 sec latency 71.012 ms 2024-09-13T13:04:55.752 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:55 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 101/1614 objects degraded (6.258%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:55.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:55 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 101/1614 objects degraded (6.258%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T13:04:56.030 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:55 smithi042 podman[75778]: 2024-09-13 13:04:55.79749227 +0000 UTC m=+1.134710719 container remove 4b98d2c08f101faa507064fb6cd8cb4c7342b2d6ece5f6a13f672074a7ee411e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:04:56.030 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:55 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T13:04:56.031 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:55 smithi042 systemd[1]: Stopped Ceph osd.3 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:04:56.031 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:55 smithi042 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.3.service: Consumed 3min 55.782s CPU time. 2024-09-13T13:04:56.303 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 systemd[1]: Starting Ceph osd.3 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:04:56.574 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:56 smithi042 ceph-mon[63970]: pgmap v171: 97 pgs: 7 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.5 MiB/s wr, 473 op/s; 101/1614 objects degraded (6.258%) 2024-09-13T13:04:56.574 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 podman[75888]: 2024-09-13 13:04:56.271396772 +0000 UTC m=+0.019195043 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:04:56.574 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 podman[75888]: 2024-09-13 13:04:56.433171783 +0000 UTC m=+0.180970048 container create ef06e9a3c25c7ecb19eb3449243adef61a00bd65fa88e7edabcb9350b2d31a6a (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True) 2024-09-13T13:04:56.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 203824 9.32 MB/sec execute 421 sec latency 70.107 ms 2024-09-13T13:04:56.832 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 podman[75888]: 2024-09-13 13:04:56.820573534 +0000 UTC m=+0.568371804 container init ef06e9a3c25c7ecb19eb3449243adef61a00bd65fa88e7edabcb9350b2d31a6a (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-13T13:04:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:56 smithi046 ceph-mon[55972]: pgmap v171: 97 pgs: 7 active+undersized, 8 stale+active+clean, 11 active+undersized+degraded, 71 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.5 MiB/s wr, 473 op/s; 101/1614 objects degraded (6.258%) 2024-09-13T13:04:57.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 podman[75888]: 2024-09-13 13:04:56.830132468 +0000 UTC m=+0.577930729 container start ef06e9a3c25c7ecb19eb3449243adef61a00bd65fa88e7edabcb9350b2d31a6a (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T13:04:57.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 podman[75888]: 2024-09-13 13:04:56.979409793 +0000 UTC m=+0.727208050 container attach ef06e9a3c25c7ecb19eb3449243adef61a00bd65fa88e7edabcb9350b2d31a6a (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T13:04:57.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:57.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 bash[75888]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:57.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:57.101 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:56 smithi042 bash[75888]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:04:57.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 204051 9.32 MB/sec execute 422 sec latency 105.287 ms 2024-09-13T13:04:58.658 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:58 smithi042 ceph-mon[63970]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.7 MiB/s wr, 239 op/s; 205/1581 objects degraded (12.966%) 2024-09-13T13:04:58.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 204338 9.32 MB/sec execute 423 sec latency 84.294 ms 2024-09-13T13:04:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:58 smithi046 ceph-mon[55972]: pgmap v172: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 355 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 2.7 MiB/s wr, 239 op/s; 205/1581 objects degraded (12.966%) 2024-09-13T13:04:59.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 204652 9.31 MB/sec execute 424 sec latency 75.271 ms 2024-09-13T13:04:59.932 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:04:59.932 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:04:59.932 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:04:59 smithi042 ceph-mon[63970]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s wr, 434 op/s; 208/1590 objects degraded (13.082%) 2024-09-13T13:05:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:04:59 smithi046 ceph-mon[55972]: pgmap v173: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 4.2 MiB/s wr, 434 op/s; 208/1590 objects degraded (13.082%) 2024-09-13T13:05:00.209 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:59 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T13:05:00.209 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:59 smithi042 bash[75888]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T13:05:00.209 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:59 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T13:05:00.209 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:04:59 smithi042 bash[75888]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T13:05:00.477 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T13:05:00.477 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 bash[75888]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 bash[75888]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 bash[75888]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 bash[75888]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate[75902]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 bash[75888]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T13:05:00.478 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:00 smithi042 podman[75888]: 2024-09-13 13:05:00.372767738 +0000 UTC m=+4.120565997 container died ef06e9a3c25c7ecb19eb3449243adef61a00bd65fa88e7edabcb9350b2d31a6a (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-13T13:05:00.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 204876 9.30 MB/sec execute 425 sec latency 54.358 ms 2024-09-13T13:05:01.529 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:01 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 210/1599 objects degraded (13.133%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:01.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 205075 9.30 MB/sec execute 426 sec latency 57.458 ms 2024-09-13T13:05:01.780 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:01 smithi042 podman[76082]: 2024-09-13 13:05:01.532177461 +0000 UTC m=+1.148632256 container remove ef06e9a3c25c7ecb19eb3449243adef61a00bd65fa88e7edabcb9350b2d31a6a (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:05:01.780 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:01 smithi042 podman[76138]: 2024-09-13 13:05:01.670134934 +0000 UTC m=+0.019210032 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:05:01.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:01 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 210/1599 objects degraded (13.133%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:02.031 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:01 smithi042 podman[76138]: 2024-09-13 13:05:01.79718448 +0000 UTC m=+0.146259577 container create fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:05:02.300 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:02 smithi042 podman[76138]: 2024-09-13 13:05:02.210033971 +0000 UTC m=+0.559109081 container init fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-13T13:05:02.300 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:02 smithi042 podman[76138]: 2024-09-13 13:05:02.218704227 +0000 UTC m=+0.567779327 container start fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:05:02.300 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:02 smithi042 ceph-osd[76156]: -- 172.21.15.42:0/2386002247 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d43de9c1e0 con 0x55d43de28000 2024-09-13T13:05:02.559 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:02 smithi042 ceph-mon[63970]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.2 MiB/s wr, 348 op/s; 210/1599 objects degraded (13.133%) 2024-09-13T13:05:02.559 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:02 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:02.559 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:02 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:02.559 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:02 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:02.559 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:02 smithi042 bash[76138]: fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 2024-09-13T13:05:02.560 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:02 smithi042 systemd[1]: Started Ceph osd.3 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:05:02.661 INFO:teuthology.orchestra.run.smithi042.stdout: 5 205375 9.29 MB/sec execute 427 sec latency 77.855 ms 2024-09-13T13:05:02.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:02 smithi046 ceph-mon[55972]: pgmap v174: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 357 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 3.1 KiB/s rd, 3.2 MiB/s wr, 348 op/s; 210/1599 objects degraded (13.133%) 2024-09-13T13:05:02.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:02 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:02.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:02 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:02.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:02 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:03.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 205769 9.29 MB/sec execute 428 sec latency 42.753 ms 2024-09-13T13:05:04.341 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:04 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:05:04.093+0000 7fecc22e9640 -1 Falling back to public interface 2024-09-13T13:05:04.596 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:04 smithi042 ceph-mon[63970]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.2 MiB/s wr, 331 op/s; 215/1629 objects degraded (13.198%) 2024-09-13T13:05:04.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 206162 9.31 MB/sec execute 429 sec latency 39.667 ms 2024-09-13T13:05:04.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:04 smithi046 ceph-mon[55972]: pgmap v175: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 2.9 KiB/s rd, 3.2 MiB/s wr, 331 op/s; 215/1629 objects degraded (13.198%) 2024-09-13T13:05:05.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 206523 9.30 MB/sec execute 430 sec latency 43.194 ms 2024-09-13T13:05:06.632 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:06 smithi042 ceph-mon[63970]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.1 MiB/s wr, 398 op/s; 216/1632 objects degraded (13.235%) 2024-09-13T13:05:06.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 206978 9.30 MB/sec execute 431 sec latency 35.135 ms 2024-09-13T13:05:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:06 smithi046 ceph-mon[55972]: pgmap v176: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 356 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 2.6 KiB/s rd, 4.1 MiB/s wr, 398 op/s; 216/1632 objects degraded (13.235%) 2024-09-13T13:05:07.644 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:07 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 227/1674 objects degraded (13.560%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:07.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 207360 9.31 MB/sec execute 432 sec latency 39.722 ms 2024-09-13T13:05:07.866 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:07 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 227/1674 objects degraded (13.560%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:08.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 207703 9.31 MB/sec execute 433 sec latency 55.781 ms 2024-09-13T13:05:08.671 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:08 smithi042 ceph-mon[63970]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s; 227/1674 objects degraded (13.560%) 2024-09-13T13:05:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:08 smithi046 ceph-mon[55972]: pgmap v177: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 358 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 394 op/s; 227/1674 objects degraded (13.560%) 2024-09-13T13:05:09.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 208017 9.31 MB/sec execute 434 sec latency 69.443 ms 2024-09-13T13:05:10.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 208311 9.30 MB/sec execute 435 sec latency 65.758 ms 2024-09-13T13:05:10.711 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:10 smithi042 ceph-mon[63970]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 363 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.3 MiB/s wr, 549 op/s; 225/1662 objects degraded (13.538%) 2024-09-13T13:05:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:10 smithi046 ceph-mon[55972]: pgmap v178: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 363 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 4.9 KiB/s rd, 5.3 MiB/s wr, 549 op/s; 225/1662 objects degraded (13.538%) 2024-09-13T13:05:11.225 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:11 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:05:11.092+0000 7fecc22e9640 -1 osd.3 0 read_superblock omap replica is missing. 2024-09-13T13:05:11.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 208566 9.30 MB/sec execute 436 sec latency 41.197 ms 2024-09-13T13:05:11.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:11 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:11.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:11 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:11.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:11 smithi042 ceph-mon[63970]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 363 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.2 MiB/s wr, 421 op/s; 218/1650 objects degraded (13.212%) 2024-09-13T13:05:11.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:11 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:11.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:11 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:11 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:11 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:11 smithi046 ceph-mon[55972]: pgmap v179: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 363 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 8.6 KiB/s rd, 4.2 MiB/s wr, 421 op/s; 218/1650 objects degraded (13.212%) 2024-09-13T13:05:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:11 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:12.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:11 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:12.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 208942 9.30 MB/sec execute 437 sec latency 42.709 ms 2024-09-13T13:05:12.747 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:12 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:05:12.716+0000 7fecc22e9640 -1 osd.3 113 log_to_monitors true 2024-09-13T13:05:13.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:12 smithi042 ceph-mon[63970]: from='osd.3 [v2:172.21.15.42:6826/2295394418,v1:172.21.15.42:6827/2295394418]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T13:05:13.003 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:12 smithi042 ceph-mon[63970]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T13:05:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:12 smithi046 ceph-mon[55972]: from='osd.3 [v2:172.21.15.42:6826/2295394418,v1:172.21.15.42:6827/2295394418]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T13:05:13.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:12 smithi046 ceph-mon[55972]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-13T13:05:13.510 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:05:13.264+0000 7fecb608b640 -1 osd.3 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:05:13.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 209262 9.30 MB/sec execute 438 sec latency 71.594 ms 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: from='osd.3 [v2:172.21.15.42:6826/2295394418,v1:172.21.15.42:6827/2295394418]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: from='osd.3 ' entity='osd.3' 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 209/1623 objects degraded (12.877%) 2024-09-13T13:05:14.033 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:13 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: osdmap e116: 8 total, 7 up, 8 in 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: from='osd.3 [v2:172.21.15.42:6826/2295394418,v1:172.21.15.42:6827/2295394418]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi042", "root=default"]} : dispatch 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: from='osd.3 ' entity='osd.3' 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: pgmap v181: 97 pgs: 14 active+undersized, 27 active+undersized+degraded, 56 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 5.0 MiB/s wr, 503 op/s; 209/1623 objects degraded (12.877%) 2024-09-13T13:05:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:13 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:14.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 209608 9.30 MB/sec execute 439 sec latency 328.421 ms 2024-09-13T13:05:15.223 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:14 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:05:15.223 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:14 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 209/1623 objects degraded (12.877%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:15.223 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:14 smithi042 ceph-mon[63970]: osd.3 [v2:172.21.15.42:6826/2295394418,v1:172.21.15.42:6827/2295394418] boot 2024-09-13T13:05:15.223 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:14 smithi042 ceph-mon[63970]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T13:05:15.223 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T13:05:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:14 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:05:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:14 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 209/1623 objects degraded (12.877%), 27 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:14 smithi046 ceph-mon[55972]: osd.3 [v2:172.21.15.42:6826/2295394418,v1:172.21.15.42:6827/2295394418] boot 2024-09-13T13:05:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:14 smithi046 ceph-mon[55972]: osdmap e117: 8 total, 8 up, 8 in 2024-09-13T13:05:15.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-13T13:05:15.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 209621 9.28 MB/sec execute 440 sec latency 1328.470 ms 2024-09-13T13:05:16.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:16 smithi042 ceph-mon[63970]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T13:05:16.538 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:16 smithi042 ceph-mon[63970]: pgmap v184: 97 pgs: 10 peering, 9 active+undersized, 22 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.4 MiB/s wr, 361 op/s; 154/1644 objects degraded (9.367%) 2024-09-13T13:05:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:16 smithi046 ceph-mon[55972]: osdmap e118: 8 total, 8 up, 8 in 2024-09-13T13:05:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:16 smithi046 ceph-mon[55972]: pgmap v184: 97 pgs: 10 peering, 9 active+undersized, 22 active+undersized+degraded, 56 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.3 KiB/s rd, 3.4 MiB/s wr, 361 op/s; 154/1644 objects degraded (9.367%) 2024-09-13T13:05:16.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 209656 9.26 MB/sec execute 441 sec latency 2223.937 ms 2024-09-13T13:05:17.303 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:17 smithi042 ceph-mon[63970]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T13:05:17.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:17 smithi046 ceph-mon[55972]: osdmap e119: 8 total, 8 up, 8 in 2024-09-13T13:05:17.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 209932 9.26 MB/sec execute 442 sec latency 55.947 ms 2024-09-13T13:05:18.315 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: pgmap v187: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 4 active+undersized, 8 active+undersized+degraded, 67 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 154/1641 objects degraded (9.385%); 54 KiB/s, 4 objects/s recovering 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:18.316 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: osdmap e120: 8 total, 8 up, 8 in 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: pgmap v187: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 3 remapped+peering, 10 peering, 4 active+undersized, 8 active+undersized+degraded, 67 active+clean; 357 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 4.5 KiB/s rd, 5.3 MiB/s wr, 528 op/s; 154/1641 objects degraded (9.385%); 54 KiB/s, 4 objects/s recovering 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:05:18.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:18.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:18.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:18.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:18.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:18.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T13:05:18.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 210331 9.26 MB/sec execute 443 sec latency 713.702 ms 2024-09-13T13:05:19.347 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:19 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T13:05:19.347 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:19 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T13:05:19.347 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:19 smithi042 ceph-mon[63970]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T13:05:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:19 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T13:05:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:19 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-09-13T13:05:19.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:19 smithi046 ceph-mon[55972]: osdmap e121: 8 total, 8 up, 8 in 2024-09-13T13:05:19.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 210573 9.25 MB/sec execute 444 sec latency 892.165 ms 2024-09-13T13:05:20.390 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:20 smithi042 ceph-mon[63970]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T13:05:20.390 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:20 smithi042 ceph-mon[63970]: pgmap v190: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 5 peering, 84 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 407 op/s; 112/1626 objects degraded (6.888%); 230 KiB/s, 31 objects/s recovering 2024-09-13T13:05:20.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:20 smithi046 ceph-mon[55972]: osdmap e122: 8 total, 8 up, 8 in 2024-09-13T13:05:20.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:20 smithi046 ceph-mon[55972]: pgmap v190: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 5 peering, 84 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.0 KiB/s rd, 4.5 MiB/s wr, 407 op/s; 112/1626 objects degraded (6.888%); 230 KiB/s, 31 objects/s recovering 2024-09-13T13:05:20.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 210802 9.24 MB/sec execute 445 sec latency 745.261 ms 2024-09-13T13:05:21.428 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:21 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 112/1626 objects degraded (6.888%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:21.428 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:21 smithi042 ceph-mon[63970]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T13:05:21.613 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:05:21.616 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:21 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 112/1626 objects degraded (6.888%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:21.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:21 smithi046 ceph-mon[55972]: osdmap e123: 8 total, 8 up, 8 in 2024-09-13T13:05:21.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 211123 9.24 MB/sec execute 446 sec latency 782.600 ms 2024-09-13T13:05:22.440 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:22 smithi042 ceph-mon[63970]: pgmap v192: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 peering, 85 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.2 MiB/s wr, 382 op/s; 112/1614 objects degraded (6.939%); 1.2 MiB/s, 27 objects/s recovering 2024-09-13T13:05:22.440 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:22 smithi042 ceph-mon[63970]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T13:05:22.440 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:22 smithi042 ceph-mon[63970]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:22 smithi046 ceph-mon[55972]: pgmap v192: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 peering, 85 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.2 MiB/s wr, 382 op/s; 112/1614 objects degraded (6.939%); 1.2 MiB/s, 27 objects/s recovering 2024-09-13T13:05:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:22 smithi046 ceph-mon[55972]: osdmap e124: 8 total, 8 up, 8 in 2024-09-13T13:05:22.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:22 smithi046 ceph-mon[55972]: from='client.34334 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:22.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 211505 9.24 MB/sec execute 447 sec latency 51.740 ms 2024-09-13T13:05:22.741 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:05:22.741 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (6m) 11s ago 18m 19.8M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:05:22.741 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (3m) 11s ago 18m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:05:22.741 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (3m) 3m ago 15m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (6m) 11s ago 15m 45.1M - dad864ee21e9 7144014c89b8 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (10m) 11s ago 10m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (10m) 3m ago 10m 15.6M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (5m) 11s ago 20m 519M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (5m) 3m ago 15m 453M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (4m) 11s ago 20m 56.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (4m) 3m ago 15m 40.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (9m) 11s ago 9m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (9m) 3m ago 9m 88.2M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (6m) 11s ago 15m 10.7M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (6m) 3m ago 15m 11.4M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (2m) 11s ago 14m 335M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (2m) 11s ago 14m 254M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:05:22.742 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (71s) 11s ago 13m 179M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:05:22.743 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (21s) 11s ago 13m 11.9M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:05:22.743 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (12m) 3m ago 12m 503M 2705M 17.2.0 e1d6a67b021e 9748c196b9c1 2024-09-13T13:05:22.743 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (12m) 3m ago 12m 560M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:05:22.743 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (11m) 3m ago 11m 482M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:05:22.743 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (11m) 3m ago 11m 376M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:05:22.743 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (5m) 11s ago 15m 55.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:05:23.325 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 4 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:23.326 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:05:23.327 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:05:23.531 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:23 smithi042 ceph-mon[63970]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:23.531 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:23 smithi042 ceph-mon[63970]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:23 smithi046 ceph-mon[55972]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:23.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:23 smithi046 ceph-mon[55972]: from='client.34342 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:23.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 211804 9.24 MB/sec execute 448 sec latency 65.407 ms 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:05:23.919 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:05:23.920 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:05:23.920 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "10/23 daemons upgraded", 2024-09-13T13:05:23.920 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:05:23.920 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:05:23.920 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:05:24.550 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:24 smithi042 ceph-mon[63970]: pgmap v194: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 94 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.5 MiB/s wr, 355 op/s; 5/1596 objects degraded (0.313%); 1.8 MiB/s, 42 objects/s recovering 2024-09-13T13:05:24.550 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:24 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2243335666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:24 smithi046 ceph-mon[55972]: pgmap v194: 97 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 peering, 94 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 3.5 MiB/s wr, 355 op/s; 5/1596 objects degraded (0.313%); 1.8 MiB/s, 42 objects/s recovering 2024-09-13T13:05:24.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:24 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2243335666' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:24.643 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN Degraded data redundancy: 5/1596 objects degraded (0.313%), 1 pg degraded 2024-09-13T13:05:24.644 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 5/1596 objects degraded (0.313%), 1 pg degraded 2024-09-13T13:05:24.644 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.c is active+recovering+undersized+degraded+remapped, acting [5,6] 2024-09-13T13:05:24.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 212158 9.24 MB/sec execute 449 sec latency 40.703 ms 2024-09-13T13:05:25.588 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:25 smithi042 ceph-mon[63970]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:25.589 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:25 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1605219935' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:05:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:25 smithi046 ceph-mon[55972]: from='client.34348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:25.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:25 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1605219935' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:05:25.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 212542 9.25 MB/sec execute 450 sec latency 51.618 ms 2024-09-13T13:05:26.529 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:26 smithi042 ceph-mon[63970]: pgmap v195: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.0 MiB/s wr, 216 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-09-13T13:05:26.529 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:26 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1596 objects degraded (0.313%), 1 pg degraded) 2024-09-13T13:05:26.529 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:26 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:05:26.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:26 smithi046 ceph-mon[55972]: pgmap v195: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.0 MiB/s wr, 216 op/s; 1.5 MiB/s, 27 objects/s recovering 2024-09-13T13:05:26.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:26 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/1596 objects degraded (0.313%), 1 pg degraded) 2024-09-13T13:05:26.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:26 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:05:26.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 212854 9.25 MB/sec execute 451 sec latency 44.976 ms 2024-09-13T13:05:27.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 213209 9.26 MB/sec execute 452 sec latency 71.032 ms 2024-09-13T13:05:28.599 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:28 smithi042 ceph-mon[63970]: pgmap v196: 97 pgs: 97 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.6 MiB/s wr, 324 op/s; 1.1 MiB/s, 20 objects/s recovering 2024-09-13T13:05:28.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:28 smithi046 ceph-mon[55972]: pgmap v196: 97 pgs: 97 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 3.6 MiB/s wr, 324 op/s; 1.1 MiB/s, 20 objects/s recovering 2024-09-13T13:05:28.662 INFO:teuthology.orchestra.run.smithi042.stdout: 5 213361 9.24 MB/sec execute 453 sec latency 67.310 ms 2024-09-13T13:05:29.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 213627 9.23 MB/sec execute 454 sec latency 61.998 ms 2024-09-13T13:05:29.907 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:29.907 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:29.907 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:29 smithi042 ceph-mon[63970]: pgmap v197: 97 pgs: 97 active+clean; 361 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 503 op/s; 1.0 MiB/s, 18 objects/s recovering 2024-09-13T13:05:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:29 smithi046 ceph-mon[55972]: pgmap v197: 97 pgs: 97 active+clean; 361 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 5.4 MiB/s wr, 503 op/s; 1.0 MiB/s, 18 objects/s recovering 2024-09-13T13:05:30.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 213875 9.22 MB/sec execute 455 sec latency 79.712 ms 2024-09-13T13:05:31.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 214079 9.22 MB/sec execute 456 sec latency 69.015 ms 2024-09-13T13:05:32.443 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:32 smithi042 ceph-mon[63970]: pgmap v198: 97 pgs: 97 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 442 op/s; 514 KiB/s, 16 objects/s recovering 2024-09-13T13:05:32.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 214464 9.22 MB/sec execute 457 sec latency 50.726 ms 2024-09-13T13:05:32.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:32 smithi046 ceph-mon[55972]: pgmap v198: 97 pgs: 97 active+clean; 356 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.8 MiB/s wr, 442 op/s; 514 KiB/s, 16 objects/s recovering 2024-09-13T13:05:33.454 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:33 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T13:05:33.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 214780 9.21 MB/sec execute 458 sec latency 51.414 ms 2024-09-13T13:05:33.756 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:33 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-13T13:05:34.460 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:34 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T13:05:34.460 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:34 smithi042 ceph-mon[63970]: Upgrade: osd.4 is safe to restart 2024-09-13T13:05:34.460 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:34 smithi042 ceph-mon[63970]: pgmap v199: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 389 op/s; 428 KiB/s, 13 objects/s recovering 2024-09-13T13:05:34.460 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:34 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:34.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 215375 9.22 MB/sec execute 459 sec latency 39.748 ms 2024-09-13T13:05:34.836 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:34 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T13:05:34.837 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:34 smithi046 ceph-mon[55972]: Upgrade: osd.4 is safe to restart 2024-09-13T13:05:34.837 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:34 smithi046 ceph-mon[55972]: pgmap v199: 97 pgs: 97 active+clean; 354 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 4.4 MiB/s wr, 389 op/s; 428 KiB/s, 13 objects/s recovering 2024-09-13T13:05:34.837 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:34 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:35.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 215578 9.22 MB/sec execute 460 sec latency 72.377 ms 2024-09-13T13:05:36.034 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:35 smithi042 ceph-mon[63970]: Upgrade: Updating osd.4 2024-09-13T13:05:36.034 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:36.034 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T13:05:36.035 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:05:36.035 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:35 smithi042 ceph-mon[63970]: Deploying daemon osd.4 on smithi046 2024-09-13T13:05:36.035 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:35 smithi042 ceph-mon[63970]: pgmap v200: 97 pgs: 97 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T13:05:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:35 smithi046 ceph-mon[55972]: Upgrade: Updating osd.4 2024-09-13T13:05:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-13T13:05:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:05:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:35 smithi046 ceph-mon[55972]: Deploying daemon osd.4 on smithi046 2024-09-13T13:05:36.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:35 smithi046 ceph-mon[55972]: pgmap v200: 97 pgs: 97 active+clean; 359 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.4 KiB/s rd, 5.1 MiB/s wr, 534 op/s; 0 B/s, 1 objects/s recovering 2024-09-13T13:05:36.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 215886 9.22 MB/sec execute 461 sec latency 61.616 ms 2024-09-13T13:05:37.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 216124 9.21 MB/sec execute 462 sec latency 57.174 ms 2024-09-13T13:05:38.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:38 smithi046 systemd[1]: Stopping Ceph osd.4 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:05:38.646 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:38 smithi042 ceph-mon[63970]: pgmap v201: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.5 MiB/s wr, 484 op/s 2024-09-13T13:05:38.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 216350 9.21 MB/sec execute 463 sec latency 80.196 ms 2024-09-13T13:05:38.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:38 smithi046 ceph-mon[55972]: pgmap v201: 97 pgs: 97 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.3 KiB/s rd, 4.5 MiB/s wr, 484 op/s 2024-09-13T13:05:38.868 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:38 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[36533]: 2024-09-13T13:05:38.469+0000 7fe1a74ad700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:05:38.868 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:38 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[36533]: 2024-09-13T13:05:38.469+0000 7fe1a74ad700 -1 osd.4 124 *** Got signal Terminated *** 2024-09-13T13:05:38.868 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:38 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[36533]: 2024-09-13T13:05:38.469+0000 7fe1a74ad700 -1 osd.4 124 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:05:39.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 216607 9.21 MB/sec execute 464 sec latency 219.332 ms 2024-09-13T13:05:39.677 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:39 smithi042 ceph-mon[63970]: osd.4 marked itself down and dead 2024-09-13T13:05:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:39 smithi046 ceph-mon[55972]: osd.4 marked itself down and dead 2024-09-13T13:05:40.303 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:40 smithi046 podman[59427]: 2024-09-13 13:05:40.009814217 +0000 UTC m=+1.660006220 container died 9748c196b9c13757cb837ace6b63470462851b14c0e2ca263556f810b5d0d74f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.openshift.expose-services=, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, release=754, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, version=8, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, vcs-type=git, distribution-scope=public, GIT_BRANCH=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, vendor=Red Hat, Inc.) 2024-09-13T13:05:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:40 smithi046 ceph-mon[55972]: pgmap v202: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 527 op/s 2024-09-13T13:05:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:40 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:05:40.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:40 smithi046 ceph-mon[55972]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T13:05:40.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:40 smithi046 podman[59427]: 2024-09-13 13:05:40.393856064 +0000 UTC m=+2.044048067 container cleanup 9748c196b9c13757cb837ace6b63470462851b14c0e2ca263556f810b5d0d74f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, RELEASE=HEAD, name=centos-stream, version=8, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , release=754, com.redhat.component=centos-stream-container, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-09-13T13:05:40.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:40 smithi046 bash[59427]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4 2024-09-13T13:05:40.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 216695 9.19 MB/sec execute 465 sec latency 1047.114 ms 2024-09-13T13:05:40.688 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:40 smithi042 ceph-mon[63970]: pgmap v202: 97 pgs: 97 active+clean; 356 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 5.2 MiB/s wr, 527 op/s 2024-09-13T13:05:40.688 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:40 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:05:40.688 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:40 smithi042 ceph-mon[63970]: osdmap e125: 8 total, 7 up, 8 in 2024-09-13T13:05:41.069 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:40 smithi046 podman[59445]: 2024-09-13 13:05:40.687578345 +0000 UTC m=+0.669159807 container remove 9748c196b9c13757cb837ace6b63470462851b14c0e2ca263556f810b5d0d74f (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, ceph=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.expose-services=, com.redhat.component=centos-stream-container, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux ) 2024-09-13T13:05:41.342 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:41 smithi046 podman[59529]: 2024-09-13 13:05:40.972151047 +0000 UTC m=+0.017130230 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:05:41.342 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:41 smithi046 podman[59529]: 2024-09-13 13:05:41.088569847 +0000 UTC m=+0.133549020 container create a780d59d611e9fee145f46473441f43f896cf48ca233f0242815f27eadd81e82 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-13T13:05:41.342 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:41 smithi046 podman[59529]: 2024-09-13 13:05:41.341544968 +0000 UTC m=+0.386524140 container init a780d59d611e9fee145f46473441f43f896cf48ca233f0242815f27eadd81e82 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:05:41.478 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:41 smithi042 ceph-mon[63970]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T13:05:41.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:41 smithi046 ceph-mon[55972]: osdmap e126: 8 total, 7 up, 8 in 2024-09-13T13:05:41.618 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:41 smithi046 podman[59529]: 2024-09-13 13:05:41.345274825 +0000 UTC m=+0.390254002 container start a780d59d611e9fee145f46473441f43f896cf48ca233f0242815f27eadd81e82 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:05:41.618 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:41 smithi046 podman[59529]: 2024-09-13 13:05:41.433101796 +0000 UTC m=+0.478080969 container attach a780d59d611e9fee145f46473441f43f896cf48ca233f0242815f27eadd81e82 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-09-13T13:05:41.618 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:41 smithi046 podman[59529]: 2024-09-13 13:05:41.562137323 +0000 UTC m=+0.607116498 container died a780d59d611e9fee145f46473441f43f896cf48ca233f0242815f27eadd81e82 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-13T13:05:41.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 216892 9.19 MB/sec execute 466 sec latency 62.527 ms 2024-09-13T13:05:42.578 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:42 smithi042 ceph-mon[63970]: pgmap v205: 97 pgs: 4 active+undersized, 10 stale+active+clean, 5 active+undersized+degraded, 78 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 522 op/s; 41/1605 objects degraded (2.555%) 2024-09-13T13:05:42.578 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:42 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 41/1605 objects degraded (2.555%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:42.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 217126 9.18 MB/sec execute 467 sec latency 59.103 ms 2024-09-13T13:05:42.742 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 podman[59551]: 2024-09-13 13:05:42.487803344 +0000 UTC m=+0.913471026 container remove a780d59d611e9fee145f46473441f43f896cf48ca233f0242815f27eadd81e82 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-13T13:05:42.743 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T13:05:42.743 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 systemd[1]: Stopped Ceph osd.4 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:05:42.743 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.4.service: Consumed 4min 1.611s CPU time. 2024-09-13T13:05:42.743 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:42 smithi046 ceph-mon[55972]: pgmap v205: 97 pgs: 4 active+undersized, 10 stale+active+clean, 5 active+undersized+degraded, 78 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.9 MiB/s wr, 522 op/s; 41/1605 objects degraded (2.555%) 2024-09-13T13:05:42.743 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:42 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 41/1605 objects degraded (2.555%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:42.997 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 systemd[1]: Starting Ceph osd.4 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:05:42.997 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 podman[59648]: 2024-09-13 13:05:42.8708632 +0000 UTC m=+0.018641316 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:05:43.250 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:42 smithi046 podman[59648]: 2024-09-13 13:05:42.997235114 +0000 UTC m=+0.145013216 container create db47feea44c9c9ac6e5a03df22c64088a3fd9e68ccd7f0b1e21d44be8bc1abcb (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T13:05:43.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 podman[59648]: 2024-09-13 13:05:43.250036583 +0000 UTC m=+0.397814687 container init db47feea44c9c9ac6e5a03df22c64088a3fd9e68ccd7f0b1e21d44be8bc1abcb (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-13T13:05:43.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 podman[59648]: 2024-09-13 13:05:43.253856949 +0000 UTC m=+0.401635058 container start db47feea44c9c9ac6e5a03df22c64088a3fd9e68ccd7f0b1e21d44be8bc1abcb (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True) 2024-09-13T13:05:43.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 podman[59648]: 2024-09-13 13:05:43.358435182 +0000 UTC m=+0.506213286 container attach db47feea44c9c9ac6e5a03df22c64088a3fd9e68ccd7f0b1e21d44be8bc1abcb (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-13T13:05:43.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:05:43.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 bash[59648]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:05:43.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:05:43.618 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:43 smithi046 bash[59648]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:05:43.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 217314 9.17 MB/sec execute 468 sec latency 72.220 ms 2024-09-13T13:05:44.638 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:44 smithi042 ceph-mon[63970]: pgmap v206: 97 pgs: 9 active+undersized, 5 stale+active+clean, 13 active+undersized+degraded, 70 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.5 MiB/s wr, 289 op/s; 117/1602 objects degraded (7.303%) 2024-09-13T13:05:44.638 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:44.638 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:44.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 217509 9.16 MB/sec execute 469 sec latency 59.643 ms 2024-09-13T13:05:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:44 smithi046 ceph-mon[55972]: pgmap v206: 97 pgs: 9 active+undersized, 5 stale+active+clean, 13 active+undersized+degraded, 70 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 8.1 KiB/s rd, 3.5 MiB/s wr, 289 op/s; 117/1602 objects degraded (7.303%) 2024-09-13T13:05:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:05:45.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 217721 9.15 MB/sec execute 470 sec latency 86.674 ms 2024-09-13T13:05:46.053 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:45 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T13:05:46.053 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:45 smithi046 bash[59648]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T13:05:46.054 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:45 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T13:05:46.054 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:45 smithi046 bash[59648]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T13:05:46.054 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T13:05:46.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 bash[59648]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T13:05:46.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T13:05:46.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 bash[59648]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T13:05:46.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T13:05:46.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 bash[59648]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T13:05:46.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T13:05:46.368 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 bash[59648]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T13:05:46.369 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate[59664]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T13:05:46.369 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 bash[59648]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T13:05:46.369 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:46 smithi046 podman[59648]: 2024-09-13 13:05:46.084785051 +0000 UTC m=+3.232563164 container died db47feea44c9c9ac6e5a03df22c64088a3fd9e68ccd7f0b1e21d44be8bc1abcb (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux ) 2024-09-13T13:05:46.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 217960 9.14 MB/sec execute 471 sec latency 71.064 ms 2024-09-13T13:05:46.732 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:46 smithi042 ceph-mon[63970]: pgmap v207: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 395 op/s; 246/1584 objects degraded (15.530%) 2024-09-13T13:05:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:46 smithi046 ceph-mon[55972]: pgmap v207: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 357 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 3.7 KiB/s rd, 4.1 MiB/s wr, 395 op/s; 246/1584 objects degraded (15.530%) 2024-09-13T13:05:47.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:47 smithi046 podman[59817]: 2024-09-13 13:05:47.038702502 +0000 UTC m=+0.947286828 container remove db47feea44c9c9ac6e5a03df22c64088a3fd9e68ccd7f0b1e21d44be8bc1abcb (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T13:05:47.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:47 smithi046 podman[59877]: 2024-09-13 13:05:47.153245913 +0000 UTC m=+0.016505800 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:05:47.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:47 smithi046 podman[59877]: 2024-09-13 13:05:47.26449688 +0000 UTC m=+0.127756764 container create fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T13:05:47.555 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:47 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 250/1593 objects degraded (15.694%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:47.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 218152 9.13 MB/sec execute 472 sec latency 85.042 ms 2024-09-13T13:05:47.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:47 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 250/1593 objects degraded (15.694%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:47.867 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:47 smithi046 podman[59877]: 2024-09-13 13:05:47.50884144 +0000 UTC m=+0.372101324 container init fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T13:05:47.867 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:47 smithi046 podman[59877]: 2024-09-13 13:05:47.512553667 +0000 UTC m=+0.375813554 container start fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0) 2024-09-13T13:05:47.867 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:47 smithi046 ceph-osd[59896]: -- 172.21.15.46:0/2900008360 <== mon.1 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55595d78e1e0 con 0x55595d71a000 2024-09-13T13:05:48.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:48 smithi046 bash[59877]: fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de 2024-09-13T13:05:48.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:48 smithi046 systemd[1]: Started Ceph osd.4 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:05:48.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 218406 9.13 MB/sec execute 473 sec latency 54.942 ms 2024-09-13T13:05:48.840 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:48 smithi042 ceph-mon[63970]: pgmap v208: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.3 MiB/s wr, 197 op/s; 250/1593 objects degraded (15.694%) 2024-09-13T13:05:48.840 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:48 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:48.840 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:48 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:48.840 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:48 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:48.841 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:48 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:48.843 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:48 smithi046 ceph-mon[55972]: pgmap v208: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 2.3 MiB/s wr, 197 op/s; 250/1593 objects degraded (15.694%) 2024-09-13T13:05:48.843 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:48 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:48.843 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:48 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:48.843 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:48 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:48.843 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:48 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:49.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:49 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:49.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 218709 9.12 MB/sec execute 474 sec latency 56.546 ms 2024-09-13T13:05:49.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:49 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:49.867 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:49 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:05:49.409+0000 7f52c70b3640 -1 Falling back to public interface 2024-09-13T13:05:50.647 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:50 smithi042 ceph-mon[63970]: pgmap v209: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.9 MiB/s wr, 310 op/s; 255/1572 objects degraded (16.221%) 2024-09-13T13:05:50.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 218959 9.12 MB/sec execute 475 sec latency 51.079 ms 2024-09-13T13:05:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:50 smithi046 ceph-mon[55972]: pgmap v209: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.9 MiB/s wr, 310 op/s; 255/1572 objects degraded (16.221%) 2024-09-13T13:05:51.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 219162 9.11 MB/sec execute 476 sec latency 83.182 ms 2024-09-13T13:05:52.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 219473 9.12 MB/sec execute 477 sec latency 145.190 ms 2024-09-13T13:05:52.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:52 smithi042 ceph-mon[63970]: pgmap v210: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.8 MiB/s wr, 294 op/s; 251/1560 objects degraded (16.090%) 2024-09-13T13:05:52.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:52.702 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:52 smithi046 ceph-mon[55972]: pgmap v210: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 353 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 7.6 KiB/s rd, 2.8 MiB/s wr, 294 op/s; 251/1560 objects degraded (16.090%) 2024-09-13T13:05:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:53.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 219668 9.11 MB/sec execute 478 sec latency 66.293 ms 2024-09-13T13:05:54.065 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:54.065 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:53 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:54.065 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:53 smithi042 ceph-mon[63970]: pgmap v211: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.9 MiB/s wr, 280 op/s; 256/1626 objects degraded (15.744%) 2024-09-13T13:05:54.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:53 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:54.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:53 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:54.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:53 smithi046 ceph-mon[55972]: pgmap v211: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 355 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 2.9 MiB/s wr, 280 op/s; 256/1626 objects degraded (15.744%) 2024-09-13T13:05:54.663 INFO:teuthology.orchestra.run.smithi042.stdout: 5 219909 9.10 MB/sec execute 479 sec latency 61.758 ms 2024-09-13T13:05:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:54 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 256/1626 objects degraded (15.744%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:55.079 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:54 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 256/1626 objects degraded (15.744%), 29 pgs degraded (PG_DEGRADED) 2024-09-13T13:05:55.137 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:05:55.668 INFO:teuthology.orchestra.run.smithi042.stdout: 5 220241 9.11 MB/sec execute 480 sec latency 54.298 ms 2024-09-13T13:05:55.864 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:55 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:05:55.458+0000 7f52c70b3640 -1 osd.4 0 read_superblock omap replica is missing. 2024-09-13T13:05:56.114 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:55 smithi042 ceph-mon[63970]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:56.114 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:55 smithi042 ceph-mon[63970]: pgmap v212: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 362 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.6 MiB/s wr, 386 op/s; 255/1608 objects degraded (15.858%) 2024-09-13T13:05:56.114 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:55 smithi042 ceph-mon[63970]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:55 smithi046 ceph-mon[55972]: from='client.34358 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:55 smithi046 ceph-mon[55972]: pgmap v212: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 362 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.6 MiB/s wr, 386 op/s; 255/1608 objects degraded (15.858%) 2024-09-13T13:05:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:55 smithi046 ceph-mon[55972]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (7m) 45s ago 19m 19.8M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (4m) 45s ago 19m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (4m) 4s ago 16m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (7m) 45s ago 16m 45.1M - dad864ee21e9 7144014c89b8 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (10m) 45s ago 10m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (10m) 4s ago 10m 16.3M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (6m) 45s ago 20m 519M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (5m) 4s ago 16m 455M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:05:56.215 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (5m) 45s ago 20m 56.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (4m) 4s ago 15m 46.9M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (10m) 45s ago 10m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (10m) 4s ago 10m 90.9M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (7m) 45s ago 16m 10.7M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (6m) 4s ago 15m 11.1M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (3m) 45s ago 15m 335M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (2m) 45s ago 14m 254M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (104s) 45s ago 14m 179M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (54s) 45s ago 13m 11.9M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (9s) 4s ago 13m 12.2M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (12m) 4s ago 12m 512M 2705M 17.2.0 e1d6a67b021e 9673f37e42de 2024-09-13T13:05:56.216 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (12m) 4s ago 12m 432M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:05:56.217 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (12m) 4s ago 12m 698M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:05:56.217 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (5m) 45s ago 15m 55.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:05:56.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 220578 9.10 MB/sec execute 481 sec latency 54.012 ms 2024-09-13T13:05:56.946 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:05:56.946 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 4 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:05:56.947 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:05:56.948 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-09-13T13:05:56.948 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:05:56.948 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:05:56.948 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:05:57.117 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:56 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:05:56.734+0000 7f52c70b3640 -1 osd.4 124 log_to_monitors true 2024-09-13T13:05:57.546 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:05:57.546 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:05:57.546 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "11/23 daemons upgraded", 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:05:57.547 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:05:57.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 220871 9.09 MB/sec execute 482 sec latency 64.361 ms 2024-09-13T13:05:57.897 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:57.897 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:57.897 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:57.897 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='osd.4 [v2:172.21.15.46:6800/4063060710,v1:172.21.15.46:6801/4063060710]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/4217234551' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:57.898 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T13:05:58.117 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:05:57.731+0000 7f52bee5d640 -1 osd.4 124 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:05:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='client.44273 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:05:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='osd.4 [v2:172.21.15.46:6800/4063060710,v1:172.21.15.46:6801/4063060710]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/4217234551' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:58.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:05:58.120 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 253/1608 objects degraded (15.734%), 29 pgs degraded 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout: osd.4 (root=default,host=smithi046) is down 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 253/1608 objects degraded (15.734%), 29 pgs degraded 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.5 is active+undersized+degraded, acting [7,0] 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.6 is active+undersized+degraded, acting [1,6] 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.7 is active+undersized+degraded, acting [6,2] 2024-09-13T13:05:58.184 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.b is active+undersized+degraded, acting [7,5] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.d is active+undersized+degraded, acting [1,3] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.f is active+undersized+degraded, acting [0,7] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.11 is active+undersized+degraded, acting [6,1] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.13 is active+undersized+degraded, acting [0,3] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.18 is active+undersized+degraded, acting [5,7] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1a is active+undersized+degraded, acting [6,7] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1f is active+undersized+degraded, acting [0,7] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1 is active+undersized+degraded, acting [0,3] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.3 is active+undersized+degraded, acting [0,6] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.6 is active+undersized+degraded, acting [0,1] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.9 is active+undersized+degraded, acting [2,7] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.a is active+undersized+degraded, acting [6,1] 2024-09-13T13:05:58.185 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.b is active+undersized+degraded, acting [3,0] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.e is active+undersized+degraded, acting [7,1] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.f is active+undersized+degraded, acting [7,0] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.11 is active+undersized+degraded, acting [7,6] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.13 is active+undersized+degraded, acting [7,2] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.14 is active+undersized+degraded, acting [7,6] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.15 is active+undersized+degraded, acting [7,3] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.19 is active+undersized+degraded, acting [1,3] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1a is active+undersized+degraded, acting [1,2] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1b is active+undersized+degraded, acting [0,7] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1c is active+undersized+degraded, acting [5,1] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1d is active+undersized+degraded, acting [5,6] 2024-09-13T13:05:58.186 INFO:teuthology.orchestra.run.smithi042.stdout: pg 4.5 is active+undersized+degraded, acting [6,7] 2024-09-13T13:05:58.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 221219 9.10 MB/sec execute 483 sec latency 81.230 ms 2024-09-13T13:05:58.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: pgmap v213: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 253/1608 objects degraded (15.734%) 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: from='osd.4 [v2:172.21.15.46:6800/4063060710,v1:172.21.15.46:6801/4063060710]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: osdmap e127: 8 total, 7 up, 8 in 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:05:58.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:58 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2512662329' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: pgmap v213: 97 pgs: 13 active+undersized, 29 active+undersized+degraded, 55 active+clean; 360 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 253/1608 objects degraded (15.734%) 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: from='client.34374 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: from='osd.4 [v2:172.21.15.46:6800/4063060710,v1:172.21.15.46:6801/4063060710]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:05:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: osdmap e127: 8 total, 7 up, 8 in 2024-09-13T13:05:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:05:59.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:58 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2512662329' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:05:59.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 221230 9.08 MB/sec execute 484 sec latency 978.822 ms 2024-09-13T13:05:59.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:59 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:05:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:59 smithi042 ceph-mon[63970]: osd.4 [v2:172.21.15.46:6800/4063060710,v1:172.21.15.46:6801/4063060710] boot 2024-09-13T13:05:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:59 smithi042 ceph-mon[63970]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T13:05:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T13:05:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:05:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:05:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:59 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:06:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:59 smithi046 ceph-mon[55972]: osd.4 [v2:172.21.15.46:6800/4063060710,v1:172.21.15.46:6801/4063060710] boot 2024-09-13T13:06:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:59 smithi046 ceph-mon[55972]: osdmap e128: 8 total, 8 up, 8 in 2024-09-13T13:06:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-13T13:06:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:05:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:00.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 221230 9.06 MB/sec execute 485 sec latency 1978.880 ms 2024-09-13T13:06:00.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:00 smithi042 ceph-mon[63970]: pgmap v216: 97 pgs: 15 peering, 13 active+undersized, 14 active+undersized+degraded, 55 active+clean; 361 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.5 MiB/s wr, 414 op/s; 139/1626 objects degraded (8.549%) 2024-09-13T13:06:00.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:00 smithi042 ceph-mon[63970]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T13:06:00.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:00 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 139/1626 objects degraded (8.549%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:00.979 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:00 smithi042 ceph-mon[63970]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T13:06:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:00 smithi046 ceph-mon[55972]: pgmap v216: 97 pgs: 15 peering, 13 active+undersized, 14 active+undersized+degraded, 55 active+clean; 361 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 4.5 MiB/s wr, 414 op/s; 139/1626 objects degraded (8.549%) 2024-09-13T13:06:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:00 smithi046 ceph-mon[55972]: osdmap e129: 8 total, 8 up, 8 in 2024-09-13T13:06:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:00 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 139/1626 objects degraded (8.549%), 14 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:00 smithi046 ceph-mon[55972]: osdmap e130: 8 total, 8 up, 8 in 2024-09-13T13:06:01.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 221440 9.06 MB/sec execute 486 sec latency 2175.344 ms 2024-09-13T13:06:02.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 221457 9.04 MB/sec execute 487 sec latency 965.020 ms 2024-09-13T13:06:02.751 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:02 smithi042 ceph-mon[63970]: pgmap v219: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 15 peering, 1 active+recovery_wait+degraded, 11 active+undersized, 10 active+undersized+degraded, 58 active+clean; 363 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.3 MiB/s wr, 373 op/s; 143/1626 objects degraded (8.795%); 1.0 MiB/s, 56 keys/s, 0 objects/s recovering 2024-09-13T13:06:02.752 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:02 smithi042 ceph-mon[63970]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T13:06:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:02 smithi046 ceph-mon[55972]: pgmap v219: 97 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 15 peering, 1 active+recovery_wait+degraded, 11 active+undersized, 10 active+undersized+degraded, 58 active+clean; 363 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.5 KiB/s rd, 3.3 MiB/s wr, 373 op/s; 143/1626 objects degraded (8.795%); 1.0 MiB/s, 56 keys/s, 0 objects/s recovering 2024-09-13T13:06:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:02 smithi046 ceph-mon[55972]: osdmap e131: 8 total, 8 up, 8 in 2024-09-13T13:06:03.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 221714 9.04 MB/sec execute 488 sec latency 1086.528 ms 2024-09-13T13:06:03.759 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:03 smithi042 ceph-mon[63970]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T13:06:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:03 smithi046 ceph-mon[55972]: osdmap e132: 8 total, 8 up, 8 in 2024-09-13T13:06:04.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 222044 9.03 MB/sec execute 489 sec latency 955.429 ms 2024-09-13T13:06:04.805 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:04 smithi042 ceph-mon[63970]: pgmap v222: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 15 peering, 2 active+recovery_wait+degraded, 1 active+undersized+degraded, 75 active+clean; 358 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.9 MiB/s wr, 212 op/s; 101/1566 objects degraded (6.450%); 1.4 MiB/s, 56 keys/s, 13 objects/s recovering 2024-09-13T13:06:04.805 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:04 smithi042 ceph-mon[63970]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T13:06:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:04 smithi046 ceph-mon[55972]: pgmap v222: 97 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 15 peering, 2 active+recovery_wait+degraded, 1 active+undersized+degraded, 75 active+clean; 358 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.9 MiB/s wr, 212 op/s; 101/1566 objects degraded (6.450%); 1.4 MiB/s, 56 keys/s, 13 objects/s recovering 2024-09-13T13:06:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:04 smithi046 ceph-mon[55972]: osdmap e133: 8 total, 8 up, 8 in 2024-09-13T13:06:05.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 222361 9.02 MB/sec execute 490 sec latency 164.799 ms 2024-09-13T13:06:05.812 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:05 smithi042 ceph-mon[63970]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T13:06:05.812 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:05 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 101/1566 objects degraded (6.450%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:05.812 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:05 smithi042 ceph-mon[63970]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T13:06:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:05 smithi046 ceph-mon[55972]: osdmap e134: 8 total, 8 up, 8 in 2024-09-13T13:06:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:05 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 101/1566 objects degraded (6.450%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:05 smithi046 ceph-mon[55972]: osdmap e135: 8 total, 8 up, 8 in 2024-09-13T13:06:06.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 222577 9.01 MB/sec execute 491 sec latency 781.255 ms 2024-09-13T13:06:06.821 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:06 smithi042 ceph-mon[63970]: pgmap v225: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded, 87 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s wr, 252 op/s; 139/1545 objects degraded (8.997%); 2.7 MiB/s, 38 objects/s recovering 2024-09-13T13:06:06.821 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:06 smithi042 ceph-mon[63970]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T13:06:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:06 smithi046 ceph-mon[55972]: pgmap v225: 97 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 1 active+undersized+degraded, 87 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.7 MiB/s wr, 252 op/s; 139/1545 objects degraded (8.997%); 2.7 MiB/s, 38 objects/s recovering 2024-09-13T13:06:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:06 smithi046 ceph-mon[55972]: osdmap e136: 8 total, 8 up, 8 in 2024-09-13T13:06:07.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 222880 9.01 MB/sec execute 492 sec latency 722.216 ms 2024-09-13T13:06:08.628 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:08 smithi042 ceph-mon[63970]: pgmap v228: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 90 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.7 MiB/s wr, 344 op/s; 98/1566 objects degraded (6.258%); 2.5 MiB/s, 36 objects/s recovering 2024-09-13T13:06:08.628 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:08 smithi042 ceph-mon[63970]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T13:06:08.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 223226 9.02 MB/sec execute 493 sec latency 59.351 ms 2024-09-13T13:06:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:08 smithi046 ceph-mon[55972]: pgmap v228: 97 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+degraded, 90 active+clean; 359 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 2.7 MiB/s wr, 344 op/s; 98/1566 objects degraded (6.258%); 2.5 MiB/s, 36 objects/s recovering 2024-09-13T13:06:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:08 smithi046 ceph-mon[55972]: osdmap e137: 8 total, 8 up, 8 in 2024-09-13T13:06:09.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:09 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 98/1566 objects degraded (6.258%), 6 pgs degraded) 2024-09-13T13:06:09.667 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:09 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:06:09.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 223458 9.02 MB/sec execute 494 sec latency 98.873 ms 2024-09-13T13:06:09.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:09 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 98/1566 objects degraded (6.258%), 6 pgs degraded) 2024-09-13T13:06:09.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:09 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:06:10.674 INFO:teuthology.orchestra.run.smithi042.stdout: 5 223706 9.02 MB/sec execute 495 sec latency 88.530 ms 2024-09-13T13:06:10.689 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:10 smithi042 ceph-mon[63970]: pgmap v230: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 357 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.6 MiB/s wr, 293 op/s; 348 KiB/s, 32 objects/s recovering 2024-09-13T13:06:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:10 smithi046 ceph-mon[55972]: pgmap v230: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 357 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 3.6 MiB/s wr, 293 op/s; 348 KiB/s, 32 objects/s recovering 2024-09-13T13:06:11.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 223889 9.02 MB/sec execute 496 sec latency 83.153 ms 2024-09-13T13:06:12.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 224120 9.01 MB/sec execute 497 sec latency 118.759 ms 2024-09-13T13:06:12.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:12 smithi042 ceph-mon[63970]: pgmap v231: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.5 MiB/s wr, 361 op/s; 266 KiB/s, 24 objects/s recovering 2024-09-13T13:06:12.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:12 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T13:06:12.845 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:12 smithi046 ceph-mon[55972]: pgmap v231: 97 pgs: 1 active+undersized+remapped, 96 active+clean; 360 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 7.8 KiB/s rd, 3.5 MiB/s wr, 361 op/s; 266 KiB/s, 24 objects/s recovering 2024-09-13T13:06:12.845 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:12 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-13T13:06:13.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 224333 9.00 MB/sec execute 498 sec latency 77.998 ms 2024-09-13T13:06:13.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:13 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T13:06:13.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:13 smithi042 ceph-mon[63970]: Upgrade: osd.5 is safe to restart 2024-09-13T13:06:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:13 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-13T13:06:13.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:13 smithi046 ceph-mon[55972]: Upgrade: osd.5 is safe to restart 2024-09-13T13:06:14.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 224552 8.99 MB/sec execute 499 sec latency 87.562 ms 2024-09-13T13:06:14.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:14 smithi042 ceph-mon[63970]: pgmap v232: 97 pgs: 97 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 316 op/s; 203 KiB/s, 18 objects/s recovering 2024-09-13T13:06:14.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:14.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T13:06:14.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:06:14.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:14.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:14 smithi046 ceph-mon[55972]: pgmap v232: 97 pgs: 97 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.0 KiB/s rd, 3.9 MiB/s wr, 316 op/s; 203 KiB/s, 18 objects/s recovering 2024-09-13T13:06:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-13T13:06:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:06:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:15.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 224767 8.98 MB/sec execute 500 sec latency 58.776 ms 2024-09-13T13:06:15.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:15 smithi046 ceph-mon[55972]: Upgrade: Updating osd.5 2024-09-13T13:06:15.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:15 smithi046 ceph-mon[55972]: Deploying daemon osd.5 on smithi046 2024-09-13T13:06:15.870 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:15 smithi042 ceph-mon[63970]: Upgrade: Updating osd.5 2024-09-13T13:06:15.870 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:15 smithi042 ceph-mon[63970]: Deploying daemon osd.5 on smithi046 2024-09-13T13:06:16.634 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:16 smithi042 ceph-mon[63970]: pgmap v233: 97 pgs: 97 active+clean; 364 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.9 MiB/s wr, 334 op/s; 95 KiB/s, 12 objects/s recovering 2024-09-13T13:06:16.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 224999 8.98 MB/sec execute 501 sec latency 53.466 ms 2024-09-13T13:06:17.029 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:16 smithi046 ceph-mon[55972]: pgmap v233: 97 pgs: 97 active+clean; 364 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 3.2 KiB/s rd, 3.9 MiB/s wr, 334 op/s; 95 KiB/s, 12 objects/s recovering 2024-09-13T13:06:17.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 225285 8.97 MB/sec execute 502 sec latency 58.668 ms 2024-09-13T13:06:17.745 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:17 smithi046 systemd[1]: Stopping Ceph osd.5 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:06:18.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[39498]: 2024-09-13T13:06:17.742+0000 7f89e8140700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:06:18.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[39498]: 2024-09-13T13:06:17.742+0000 7f89e8140700 -1 osd.5 137 *** Got signal Terminated *** 2024-09-13T13:06:18.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[39498]: 2024-09-13T13:06:17.742+0000 7f89e8140700 -1 osd.5 137 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:06:18.669 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:18 smithi042 ceph-mon[63970]: pgmap v234: 97 pgs: 97 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 314 op/s; 84 KiB/s, 10 objects/s recovering 2024-09-13T13:06:18.669 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:18 smithi042 ceph-mon[63970]: osd.5 marked itself down and dead 2024-09-13T13:06:18.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 225678 8.97 MB/sec execute 503 sec latency 77.842 ms 2024-09-13T13:06:19.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:18 smithi046 ceph-mon[55972]: pgmap v234: 97 pgs: 97 active+clean; 359 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.8 MiB/s wr, 314 op/s; 84 KiB/s, 10 objects/s recovering 2024-09-13T13:06:19.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:18 smithi046 ceph-mon[55972]: osd.5 marked itself down and dead 2024-09-13T13:06:19.543 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:19 smithi046 podman[61457]: 2024-09-13 13:06:19.271781784 +0000 UTC m=+1.632224177 container died 9673f37e42de68c91bdff025c5b0b6568c9d851f5cafac319526ad6285118ae5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, build-date=2022-05-03T08:36:31.336870, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, version=8, vcs-type=git, io.openshift.expose-services=, release=754, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, name=centos-stream, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-09-13T13:06:19.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 225714 8.95 MB/sec execute 504 sec latency 1047.365 ms 2024-09-13T13:06:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:19 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:06:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:19 smithi046 ceph-mon[55972]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T13:06:19.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:19.867 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:19 smithi046 podman[61457]: 2024-09-13 13:06:19.633367999 +0000 UTC m=+1.993810434 container cleanup 9673f37e42de68c91bdff025c5b0b6568c9d851f5cafac319526ad6285118ae5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , version=8, io.buildah.version=1.19.8, release=754, RELEASE=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, architecture=x86_64, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb) 2024-09-13T13:06:19.867 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:19 smithi046 bash[61457]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5 2024-09-13T13:06:19.875 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:19 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:06:19.875 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:19 smithi042 ceph-mon[63970]: osdmap e138: 8 total, 7 up, 8 in 2024-09-13T13:06:19.875 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:20.242 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:19 smithi046 podman[61471]: 2024-09-13 13:06:19.910066989 +0000 UTC m=+0.635923280 container remove 9673f37e42de68c91bdff025c5b0b6568c9d851f5cafac319526ad6285118ae5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, vcs-type=git, build-date=2022-05-03T08:36:31.336870, version=8, io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True) 2024-09-13T13:06:20.506 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:20 smithi046 podman[61562]: 2024-09-13 13:06:20.144948194 +0000 UTC m=+0.016545106 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:06:20.506 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:20 smithi046 podman[61562]: 2024-09-13 13:06:20.2528373 +0000 UTC m=+0.124434200 container create fc3434ec74e86cdfd0d6cbdac15061aa6e399fa7d6660213d0597f2130b000d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-13T13:06:20.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 226044 8.95 MB/sec execute 505 sec latency 1024.892 ms 2024-09-13T13:06:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:20 smithi046 ceph-mon[55972]: pgmap v236: 97 pgs: 11 peering, 8 stale+active+clean, 78 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 385 op/s 2024-09-13T13:06:20.960 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:20 smithi046 ceph-mon[55972]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T13:06:20.960 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:20 smithi046 ceph-mon[55972]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T13:06:20.960 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:20 smithi046 podman[61562]: 2024-09-13 13:06:20.505665336 +0000 UTC m=+0.377262247 container init fc3434ec74e86cdfd0d6cbdac15061aa6e399fa7d6660213d0597f2130b000d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-13T13:06:20.960 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:20 smithi046 podman[61562]: 2024-09-13 13:06:20.509171163 +0000 UTC m=+0.380768068 container start fc3434ec74e86cdfd0d6cbdac15061aa6e399fa7d6660213d0597f2130b000d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, io.buildah.version=1.37.2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-13T13:06:20.960 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:20 smithi046 podman[61562]: 2024-09-13 13:06:20.605855929 +0000 UTC m=+0.477452840 container attach fc3434ec74e86cdfd0d6cbdac15061aa6e399fa7d6660213d0597f2130b000d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:06:20.960 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:20 smithi046 podman[61562]: 2024-09-13 13:06:20.725479153 +0000 UTC m=+0.597076054 container died fc3434ec74e86cdfd0d6cbdac15061aa6e399fa7d6660213d0597f2130b000d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-09-13T13:06:20.961 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:20 smithi042 ceph-mon[63970]: pgmap v236: 97 pgs: 11 peering, 8 stale+active+clean, 78 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.7 MiB/s wr, 385 op/s 2024-09-13T13:06:20.961 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:20 smithi042 ceph-mon[63970]: osdmap e139: 8 total, 7 up, 8 in 2024-09-13T13:06:20.961 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:20 smithi042 ceph-mon[63970]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-13T13:06:21.617 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 podman[61585]: 2024-09-13 13:06:21.336760446 +0000 UTC m=+0.605312130 container remove fc3434ec74e86cdfd0d6cbdac15061aa6e399fa7d6660213d0597f2130b000d3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-13T13:06:21.617 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.5.service: Deactivated successfully. 2024-09-13T13:06:21.618 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 systemd[1]: Stopped Ceph osd.5 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:06:21.618 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.5.service: Consumed 4min 39.788s CPU time. 2024-09-13T13:06:21.618 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 systemd[1]: Starting Ceph osd.5 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:06:21.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 226270 8.94 MB/sec execute 506 sec latency 54.037 ms 2024-09-13T13:06:21.718 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:21 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 22/1587 objects degraded (1.386%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:22.064 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:21 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 22/1587 objects degraded (1.386%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:22.065 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 podman[61684]: 2024-09-13 13:06:21.690693338 +0000 UTC m=+0.016441242 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:06:22.065 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:21 smithi046 podman[61684]: 2024-09-13 13:06:21.79507361 +0000 UTC m=+0.120821556 container create 8bdf25144752f4a1f0980b0e2f7f472b37b889cac5e8e48f9badaa8caef1cdde (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-13T13:06:22.343 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 podman[61684]: 2024-09-13 13:06:22.18932809 +0000 UTC m=+0.515075980 container init 8bdf25144752f4a1f0980b0e2f7f472b37b889cac5e8e48f9badaa8caef1cdde (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-13T13:06:22.343 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 podman[61684]: 2024-09-13 13:06:22.193110451 +0000 UTC m=+0.518858341 container start 8bdf25144752f4a1f0980b0e2f7f472b37b889cac5e8e48f9badaa8caef1cdde (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T13:06:22.343 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 podman[61684]: 2024-09-13 13:06:22.264619359 +0000 UTC m=+0.590367247 container attach 8bdf25144752f4a1f0980b0e2f7f472b37b889cac5e8e48f9badaa8caef1cdde (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:06:22.617 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:06:22.617 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 bash[61684]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:06:22.617 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:06:22.617 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:22 smithi046 bash[61684]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:06:22.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 226652 8.95 MB/sec execute 507 sec latency 40.919 ms 2024-09-13T13:06:22.739 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:22 smithi042 ceph-mon[63970]: pgmap v238: 97 pgs: 2 active+undersized, 11 peering, 6 stale+active+clean, 5 active+undersized+degraded, 73 active+clean; 358 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 432 op/s; 22/1587 objects degraded (1.386%) 2024-09-13T13:06:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:22 smithi046 ceph-mon[55972]: pgmap v238: 97 pgs: 2 active+undersized, 11 peering, 6 stale+active+clean, 5 active+undersized+degraded, 73 active+clean; 358 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.3 MiB/s wr, 432 op/s; 22/1587 objects degraded (1.386%) 2024-09-13T13:06:23.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 226916 8.95 MB/sec execute 508 sec latency 89.865 ms 2024-09-13T13:06:24.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 227288 8.96 MB/sec execute 509 sec latency 42.934 ms 2024-09-13T13:06:24.776 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:24 smithi042 ceph-mon[63970]: pgmap v239: 97 pgs: 5 active+undersized, 11 peering, 15 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 108/1569 objects degraded (6.883%) 2024-09-13T13:06:24.985 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-mon[55972]: pgmap v239: 97 pgs: 5 active+undersized, 11 peering, 15 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 7.0 KiB/s rd, 3.5 MiB/s wr, 356 op/s; 108/1569 objects degraded (6.883%) 2024-09-13T13:06:24.986 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T13:06:24.986 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T13:06:24.986 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T13:06:24.986 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T13:06:24.986 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T13:06:24.986 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T13:06:25.344 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T13:06:25.344 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T13:06:25.344 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T13:06:25.344 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-13T13:06:25.345 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T13:06:25.345 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-13T13:06:25.345 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate[61699]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T13:06:25.345 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:24 smithi046 bash[61684]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-13T13:06:25.345 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:25 smithi046 podman[61684]: 2024-09-13 13:06:25.029049774 +0000 UTC m=+3.354797675 container died 8bdf25144752f4a1f0980b0e2f7f472b37b889cac5e8e48f9badaa8caef1cdde (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:06:25.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 227539 8.96 MB/sec execute 510 sec latency 44.086 ms 2024-09-13T13:06:26.004 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:25 smithi046 ceph-mon[55972]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.7 MiB/s wr, 363 op/s; 175/1587 objects degraded (11.027%) 2024-09-13T13:06:26.004 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:25 smithi046 ceph-mon[55972]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T13:06:26.004 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:25 smithi046 podman[61860]: 2024-09-13 13:06:25.653051471 +0000 UTC m=+0.613063575 container remove 8bdf25144752f4a1f0980b0e2f7f472b37b889cac5e8e48f9badaa8caef1cdde (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-activate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:06:26.004 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:25 smithi046 podman[61918]: 2024-09-13 13:06:25.77225876 +0000 UTC m=+0.017392360 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:06:26.004 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:25 smithi046 podman[61918]: 2024-09-13 13:06:25.887334967 +0000 UTC m=+0.132468553 container create 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:06:26.049 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:25 smithi042 ceph-mon[63970]: pgmap v240: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 355 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 2.7 KiB/s rd, 3.7 MiB/s wr, 363 op/s; 175/1587 objects degraded (11.027%) 2024-09-13T13:06:26.049 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:25 smithi042 ceph-mon[63970]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-13T13:06:26.367 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:26 smithi046 podman[61918]: 2024-09-13 13:06:26.124007689 +0000 UTC m=+0.369141282 container init 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0) 2024-09-13T13:06:26.367 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:26 smithi046 podman[61918]: 2024-09-13 13:06:26.128483398 +0000 UTC m=+0.373616989 container start 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-13T13:06:26.367 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:26 smithi046 ceph-osd[61937]: -- 172.21.15.46:0/79692024 <== mon.0 v2:172.21.15.42:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558ee19701e0 con 0x558ee195e000 2024-09-13T13:06:26.367 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:26 smithi046 bash[61918]: 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 2024-09-13T13:06:26.367 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:26 smithi046 systemd[1]: Started Ceph osd.5 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:06:26.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 227802 8.95 MB/sec execute 511 sec latency 73.160 ms 2024-09-13T13:06:27.314 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:27 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:27.314 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:27 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:27.314 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:27 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:06:27.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:27 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:27.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:27 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:27.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:27 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:06:27.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 228101 8.95 MB/sec execute 512 sec latency 43.974 ms 2024-09-13T13:06:28.325 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:28 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:06:28.019+0000 7fc43b4f9640 -1 Falling back to public interface 2024-09-13T13:06:28.367 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:28 smithi042 ceph-mon[63970]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 387 op/s; 178/1605 objects degraded (11.090%) 2024-09-13T13:06:28.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:28 smithi046 ceph-mon[55972]: pgmap v241: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 5.6 KiB/s rd, 3.8 MiB/s wr, 387 op/s; 178/1605 objects degraded (11.090%) 2024-09-13T13:06:28.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 228263 8.94 MB/sec execute 513 sec latency 91.111 ms 2024-09-13T13:06:28.752 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:06:29.412 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:29 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 178/1605 objects degraded (11.090%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:29.412 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:29 smithi042 ceph-mon[63970]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:29.412 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:29.412 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:29 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 178/1605 objects degraded (11.090%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:29 smithi046 ceph-mon[55972]: from='client.34384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:29.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:29.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 228474 8.93 MB/sec execute 514 sec latency 58.502 ms 2024-09-13T13:06:29.828 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:06:29.828 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (8m) 78s ago 20m 19.8M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:06:29.828 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (4m) 78s ago 20m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (4m) 37s ago 16m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (7m) 78s ago 17m 45.1M - dad864ee21e9 7144014c89b8 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (11m) 78s ago 11m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (11m) 37s ago 11m 16.3M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (7m) 78s ago 21m 519M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (6m) 37s ago 16m 455M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (6m) 78s ago 21m 56.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (5m) 37s ago 16m 46.9M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (10m) 78s ago 10m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (10m) 37s ago 10m 90.9M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (7m) 78s ago 17m 10.7M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (7m) 37s ago 16m 11.1M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:06:29.829 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (3m) 78s ago 15m 335M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (3m) 78s ago 15m 254M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (2m) 78s ago 14m 179M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (88s) 78s ago 14m 11.9M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (42s) 37s ago 13m 12.2M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 starting - - - 2705M 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (13m) 37s ago 13m 432M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (12m) 37s ago 12m 698M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:06:29.830 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (6m) 78s ago 16m 55.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:06:30.392 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 5 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 9 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:06:30.393 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:06:30.462 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:30 smithi042 ceph-mon[63970]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:30.462 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:30 smithi042 ceph-mon[63970]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 359 op/s; 188/1659 objects degraded (11.332%) 2024-09-13T13:06:30.462 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:30.462 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:30.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 228793 8.93 MB/sec execute 515 sec latency 47.844 ms 2024-09-13T13:06:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:30 smithi046 ceph-mon[55972]: from='client.34388 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:30 smithi046 ceph-mon[55972]: pgmap v242: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.1 KiB/s rd, 4.1 MiB/s wr, 359 op/s; 188/1659 objects degraded (11.332%) 2024-09-13T13:06:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:30.946 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:06:30.946 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:06:30.946 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:06:30.946 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:06:30.946 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:06:30.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:06:30.947 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:06:30.947 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:06:30.947 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:06:30.947 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "12/23 daemons upgraded", 2024-09-13T13:06:30.948 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:06:30.948 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:06:30.948 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:06:31.414 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:31 smithi046 ceph-mon[55972]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:31.414 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:31 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2324950476' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:31.414 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:31.414 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:31.466 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:31 smithi042 ceph-mon[63970]: from='client.44293 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:31.466 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:31 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2324950476' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:31.466 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:31.466 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:31.621 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 188/1659 objects degraded (11.332%), 23 pgs degraded 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: osd.5 (root=default,host=smithi046) is down 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 188/1659 objects degraded (11.332%), 23 pgs degraded 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.2 is active+undersized+degraded, acting [1,6] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.3 is active+undersized+degraded, acting [2,7] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.b is active+undersized+degraded, acting [7,4] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.c is active+undersized+degraded, acting [2,0] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.12 is active+undersized+degraded, acting [3,7] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.14 is active+undersized+degraded, acting [6,3] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.15 is active+undersized+degraded, acting [1,0] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.16 is active+undersized+degraded, acting [6,2] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.17 is active+undersized+degraded, acting [6,2] 2024-09-13T13:06:31.622 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.18 is active+undersized+degraded, acting [4,7] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1c is active+undersized+degraded, acting [7,2] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1e is active+undersized+degraded, acting [3,0] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.2 is active+undersized+degraded, acting [3,6] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.4 is active+undersized+degraded, acting [1,2] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.5 is active+undersized+degraded, acting [3,2] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.c is active+undersized+degraded, acting [3,6] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.d is active+undersized+degraded, acting [7,6] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.10 is active+undersized+degraded, acting [6,0] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.16 is active+undersized+degraded, acting [7,1] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.17 is active+undersized+degraded, acting [0,3] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1c is active+undersized+degraded, acting [4,1] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1d is active+undersized+degraded, acting [4,6] 2024-09-13T13:06:31.623 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1f is active+undersized+degraded, acting [0,2] 2024-09-13T13:06:31.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 229199 8.93 MB/sec execute 516 sec latency 51.569 ms 2024-09-13T13:06:32.444 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:32 smithi046 ceph-mon[55972]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:32.445 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:32 smithi046 ceph-mon[55972]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 406 op/s; 190/1665 objects degraded (11.411%) 2024-09-13T13:06:32.445 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:32 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1241145017' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:06:32.484 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:32 smithi042 ceph-mon[63970]: from='client.34400 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:06:32.484 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:32 smithi042 ceph-mon[63970]: pgmap v243: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 359 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.1 MiB/s wr, 406 op/s; 190/1665 objects degraded (11.411%) 2024-09-13T13:06:32.484 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:32 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1241145017' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:06:32.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 229624 8.93 MB/sec execute 517 sec latency 39.939 ms 2024-09-13T13:06:33.498 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:33 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 181/1596 objects degraded (11.341%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:33.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 229832 8.92 MB/sec execute 518 sec latency 58.519 ms 2024-09-13T13:06:33.804 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:33 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 181/1596 objects degraded (11.341%), 23 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:34.422 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:34 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:06:34.394+0000 7fc43b4f9640 -1 osd.5 0 read_superblock omap replica is missing. 2024-09-13T13:06:34.527 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:34 smithi042 ceph-mon[63970]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 371 op/s; 181/1596 objects degraded (11.341%) 2024-09-13T13:06:34.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 230192 8.93 MB/sec execute 519 sec latency 43.461 ms 2024-09-13T13:06:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:34 smithi046 ceph-mon[55972]: pgmap v244: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 353 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 371 op/s; 181/1596 objects degraded (11.341%) 2024-09-13T13:06:35.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 230407 8.92 MB/sec execute 520 sec latency 71.374 ms 2024-09-13T13:06:35.862 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.862 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.862 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:06:35.862 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:06:35.862 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:06:35.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:35.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:35.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:35.863 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:35 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T13:06:35.866 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:06:35.695+0000 7fc43b4f9640 -1 osd.5 137 log_to_monitors true 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:35.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:06:35.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:35 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T13:06:36.675 INFO:teuthology.orchestra.run.smithi042.stdout: 5 230670 8.92 MB/sec execute 521 sec latency 55.875 ms 2024-09-13T13:06:36.874 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:36 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T13:06:36.874 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:36 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T13:06:36.874 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:36 smithi042 ceph-mon[63970]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 181/1611 objects degraded (11.235%) 2024-09-13T13:06:36.874 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:36 smithi042 ceph-mon[63970]: from='osd.5 [v2:172.21.15.46:6808/3587256383,v1:172.21.15.46:6809/3587256383]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T13:06:36.874 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:36 smithi042 ceph-mon[63970]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T13:06:37.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:06:36 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:06:36.705+0000 7fc4332a3640 -1 osd.5 137 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:06:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:36 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T13:06:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:36 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (12 PGs are or would become offline) 2024-09-13T13:06:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:36 smithi046 ceph-mon[55972]: pgmap v245: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 358 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 181/1611 objects degraded (11.235%) 2024-09-13T13:06:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:36 smithi046 ceph-mon[55972]: from='osd.5 [v2:172.21.15.46:6808/3587256383,v1:172.21.15.46:6809/3587256383]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T13:06:37.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:36 smithi046 ceph-mon[55972]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-13T13:06:37.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 230942 8.93 MB/sec execute 522 sec latency 104.503 ms 2024-09-13T13:06:37.889 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:37 smithi042 ceph-mon[63970]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T13:06:37.889 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:37 smithi042 ceph-mon[63970]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T13:06:37.889 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:37 smithi042 ceph-mon[63970]: from='osd.5 [v2:172.21.15.46:6808/3587256383,v1:172.21.15.46:6809/3587256383]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:06:37.892 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:37 smithi042 ceph-mon[63970]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:06:38.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:37 smithi046 ceph-mon[55972]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T13:06:38.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:37 smithi046 ceph-mon[55972]: osdmap e140: 8 total, 7 up, 8 in 2024-09-13T13:06:38.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:37 smithi046 ceph-mon[55972]: from='osd.5 [v2:172.21.15.46:6808/3587256383,v1:172.21.15.46:6809/3587256383]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:06:38.018 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:37 smithi046 ceph-mon[55972]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:06:38.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 231017 8.91 MB/sec execute 523 sec latency 975.150 ms 2024-09-13T13:06:38.947 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:38 smithi042 ceph-mon[63970]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 392 op/s; 181/1605 objects degraded (11.277%) 2024-09-13T13:06:38.947 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:38 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:06:38.947 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:38 smithi042 ceph-mon[63970]: osd.5 [v2:172.21.15.46:6808/3587256383,v1:172.21.15.46:6809/3587256383] boot 2024-09-13T13:06:38.947 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:38 smithi042 ceph-mon[63970]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T13:06:38.948 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T13:06:39.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:38 smithi046 ceph-mon[55972]: pgmap v247: 97 pgs: 8 active+undersized, 23 active+undersized+degraded, 66 active+clean; 360 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 4.0 MiB/s wr, 392 op/s; 181/1605 objects degraded (11.277%) 2024-09-13T13:06:39.028 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:38 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:06:39.029 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:38 smithi046 ceph-mon[55972]: osd.5 [v2:172.21.15.46:6808/3587256383,v1:172.21.15.46:6809/3587256383] boot 2024-09-13T13:06:39.029 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:38 smithi046 ceph-mon[55972]: osdmap e141: 8 total, 8 up, 8 in 2024-09-13T13:06:39.029 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-13T13:06:39.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 231217 8.90 MB/sec execute 524 sec latency 1073.654 ms 2024-09-13T13:06:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:39 smithi046 ceph-mon[55972]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T13:06:39.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:39 smithi046 ceph-mon[55972]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T13:06:39.959 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:39 smithi042 ceph-mon[63970]: osdmap e142: 8 total, 8 up, 8 in 2024-09-13T13:06:39.959 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:39 smithi042 ceph-mon[63970]: osdmap e143: 8 total, 8 up, 8 in 2024-09-13T13:06:40.676 INFO:teuthology.orchestra.run.smithi042.stdout: 5 231404 8.90 MB/sec execute 525 sec latency 648.895 ms 2024-09-13T13:06:40.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:40 smithi042 ceph-mon[63970]: pgmap v250: 97 pgs: 2 remapped+peering, 4 peering, 5 active+undersized, 15 active+undersized+degraded, 71 active+clean; 361 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 531 op/s; 119/1617 objects degraded (7.359%); 14 B/s, 0 objects/s recovering 2024-09-13T13:06:40.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:40 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 119/1617 objects degraded (7.359%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:40.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:40 smithi042 ceph-mon[63970]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T13:06:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:40 smithi046 ceph-mon[55972]: pgmap v250: 97 pgs: 2 remapped+peering, 4 peering, 5 active+undersized, 15 active+undersized+degraded, 71 active+clean; 361 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.7 MiB/s wr, 531 op/s; 119/1617 objects degraded (7.359%); 14 B/s, 0 objects/s recovering 2024-09-13T13:06:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:40 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 119/1617 objects degraded (7.359%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T13:06:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:40 smithi046 ceph-mon[55972]: osdmap e144: 8 total, 8 up, 8 in 2024-09-13T13:06:41.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 231650 8.89 MB/sec execute 526 sec latency 740.528 ms 2024-09-13T13:06:42.565 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:42 smithi042 ceph-mon[63970]: pgmap v253: 97 pgs: 2 remapped+peering, 4 peering, 3 active+undersized, 12 active+undersized+degraded, 76 active+clean; 361 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 346 op/s; 102/1632 objects degraded (6.250%); 54 KiB/s, 4 objects/s recovering 2024-09-13T13:06:42.565 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:42 smithi042 ceph-mon[63970]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T13:06:42.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 231861 8.88 MB/sec execute 527 sec latency 56.703 ms 2024-09-13T13:06:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:42 smithi046 ceph-mon[55972]: pgmap v253: 97 pgs: 2 remapped+peering, 4 peering, 3 active+undersized, 12 active+undersized+degraded, 76 active+clean; 361 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.7 KiB/s rd, 3.5 MiB/s wr, 346 op/s; 102/1632 objects degraded (6.250%); 54 KiB/s, 4 objects/s recovering 2024-09-13T13:06:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:42 smithi046 ceph-mon[55972]: osdmap e145: 8 total, 8 up, 8 in 2024-09-13T13:06:43.616 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:43 smithi042 ceph-mon[63970]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T13:06:43.616 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:43 smithi042 ceph-mon[63970]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T13:06:43.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 232119 8.88 MB/sec execute 528 sec latency 215.486 ms 2024-09-13T13:06:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:43 smithi046 ceph-mon[55972]: osdmap e146: 8 total, 8 up, 8 in 2024-09-13T13:06:43.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:43 smithi046 ceph-mon[55972]: osdmap e147: 8 total, 8 up, 8 in 2024-09-13T13:06:44.636 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:44 smithi042 ceph-mon[63970]: pgmap v256: 97 pgs: 2 remapped+peering, 95 active+clean; 364 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 99 op/s; 2.2 MiB/s, 46 objects/s recovering 2024-09-13T13:06:44.636 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:44 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1632 objects degraded (6.250%), 12 pgs degraded) 2024-09-13T13:06:44.636 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:44 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:06:44.636 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:44.636 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:44.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 232443 8.88 MB/sec execute 529 sec latency 66.991 ms 2024-09-13T13:06:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:44 smithi046 ceph-mon[55972]: pgmap v256: 97 pgs: 2 remapped+peering, 95 active+clean; 364 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 2.2 MiB/s wr, 99 op/s; 2.2 MiB/s, 46 objects/s recovering 2024-09-13T13:06:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:44 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 102/1632 objects degraded (6.250%), 12 pgs degraded) 2024-09-13T13:06:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:44 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:06:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:45.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 232729 8.87 MB/sec execute 530 sec latency 45.468 ms 2024-09-13T13:06:46.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 233133 8.87 MB/sec execute 531 sec latency 61.616 ms 2024-09-13T13:06:46.700 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:46 smithi042 ceph-mon[63970]: pgmap v258: 97 pgs: 97 active+clean; 363 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.6 MiB/s wr, 315 op/s; 1.8 MiB/s, 48 objects/s recovering 2024-09-13T13:06:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:46 smithi046 ceph-mon[55972]: pgmap v258: 97 pgs: 97 active+clean; 363 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.3 KiB/s rd, 3.6 MiB/s wr, 315 op/s; 1.8 MiB/s, 48 objects/s recovering 2024-09-13T13:06:47.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 233412 8.87 MB/sec execute 532 sec latency 52.293 ms 2024-09-13T13:06:48.516 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:48 smithi042 ceph-mon[63970]: pgmap v259: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 426 op/s; 1.4 MiB/s, 36 objects/s recovering 2024-09-13T13:06:48.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 233753 8.88 MB/sec execute 533 sec latency 87.567 ms 2024-09-13T13:06:48.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:48 smithi046 ceph-mon[55972]: pgmap v259: 97 pgs: 97 active+clean; 355 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 426 op/s; 1.4 MiB/s, 36 objects/s recovering 2024-09-13T13:06:49.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 234065 8.88 MB/sec execute 534 sec latency 49.975 ms 2024-09-13T13:06:50.546 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:50 smithi042 ceph-mon[63970]: pgmap v260: 97 pgs: 97 active+clean; 354 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 889 KiB/s, 23 objects/s recovering 2024-09-13T13:06:50.546 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:50 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T13:06:50.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 234356 8.87 MB/sec execute 535 sec latency 95.452 ms 2024-09-13T13:06:50.774 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:50 smithi046 ceph-mon[55972]: pgmap v260: 97 pgs: 97 active+clean; 354 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.7 KiB/s rd, 4.1 MiB/s wr, 376 op/s; 889 KiB/s, 23 objects/s recovering 2024-09-13T13:06:50.774 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:50 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-13T13:06:51.595 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:51 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T13:06:51.595 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:51 smithi042 ceph-mon[63970]: Upgrade: osd.6 is safe to restart 2024-09-13T13:06:51.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 234686 8.88 MB/sec execute 536 sec latency 62.307 ms 2024-09-13T13:06:51.809 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:51 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-13T13:06:51.809 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:51 smithi046 ceph-mon[55972]: Upgrade: osd.6 is safe to restart 2024-09-13T13:06:52.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:52 smithi042 ceph-mon[63970]: pgmap v261: 97 pgs: 97 active+clean; 360 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 509 op/s; 191 KiB/s, 9 objects/s recovering 2024-09-13T13:06:52.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:52.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T13:06:52.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:06:52.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 234864 8.87 MB/sec execute 537 sec latency 68.171 ms 2024-09-13T13:06:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:52 smithi046 ceph-mon[55972]: pgmap v261: 97 pgs: 97 active+clean; 360 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.4 MiB/s wr, 509 op/s; 191 KiB/s, 9 objects/s recovering 2024-09-13T13:06:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-13T13:06:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:06:53.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:53 smithi042 ceph-mon[63970]: Upgrade: Updating osd.6 2024-09-13T13:06:53.674 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:53 smithi042 ceph-mon[63970]: Deploying daemon osd.6 on smithi046 2024-09-13T13:06:53.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 235114 8.87 MB/sec execute 538 sec latency 40.312 ms 2024-09-13T13:06:53.789 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:53 smithi046 ceph-mon[55972]: Upgrade: Updating osd.6 2024-09-13T13:06:53.789 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:53 smithi046 ceph-mon[55972]: Deploying daemon osd.6 on smithi046 2024-09-13T13:06:54.680 INFO:teuthology.orchestra.run.smithi042.stdout: 5 235384 8.86 MB/sec execute 539 sec latency 55.103 ms 2024-09-13T13:06:54.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:54 smithi046 ceph-mon[55972]: pgmap v262: 97 pgs: 97 active+clean; 360 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 530 op/s; 0 B/s, 5 objects/s recovering 2024-09-13T13:06:54.685 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:54 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:54.796 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:54 smithi042 ceph-mon[63970]: pgmap v262: 97 pgs: 97 active+clean; 360 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.7 MiB/s wr, 530 op/s; 0 B/s, 5 objects/s recovering 2024-09-13T13:06:54.796 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:54 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:55.368 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:55 smithi046 systemd[1]: Stopping Ceph osd.6 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:06:55.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 235640 8.86 MB/sec execute 540 sec latency 74.959 ms 2024-09-13T13:06:55.867 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:55 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[42457]: 2024-09-13T13:06:55.655+0000 7f50f6e3e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:06:55.867 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:55 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[42457]: 2024-09-13T13:06:55.655+0000 7f50f6e3e700 -1 osd.6 147 *** Got signal Terminated *** 2024-09-13T13:06:55.867 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:55 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[42457]: 2024-09-13T13:06:55.655+0000 7f50f6e3e700 -1 osd.6 147 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:06:56.596 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:56 smithi042 ceph-mon[63970]: pgmap v263: 97 pgs: 97 active+clean; 361 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 460 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:06:56.596 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:56 smithi042 ceph-mon[63970]: osd.6 marked itself down and dead 2024-09-13T13:06:56.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 235911 8.85 MB/sec execute 541 sec latency 65.348 ms 2024-09-13T13:06:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:56 smithi046 ceph-mon[55972]: pgmap v263: 97 pgs: 97 active+clean; 361 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.4 MiB/s wr, 460 op/s; 0 B/s, 4 objects/s recovering 2024-09-13T13:06:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:56 smithi046 ceph-mon[55972]: osd.6 marked itself down and dead 2024-09-13T13:06:57.502 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:57 smithi046 podman[63508]: 2024-09-13 13:06:57.212413717 +0000 UTC m=+1.699922970 container died 6f862726cfc5a1e2b390c5bbbebf474113f9d2f92c98d6c774e4c9cb702b2b21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, ceph=True, vcs-type=git, distribution-scope=public, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container) 2024-09-13T13:06:57.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 236000 8.84 MB/sec execute 542 sec latency 1004.108 ms 2024-09-13T13:06:57.704 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:57 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:06:57.704 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:57 smithi042 ceph-mon[63970]: osdmap e148: 8 total, 7 up, 8 in 2024-09-13T13:06:57.835 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:57 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:06:57.835 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:57 smithi046 ceph-mon[55972]: osdmap e148: 8 total, 7 up, 8 in 2024-09-13T13:06:57.835 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:57 smithi046 podman[63508]: 2024-09-13 13:06:57.599666717 +0000 UTC m=+2.087175970 container cleanup 6f862726cfc5a1e2b390c5bbbebf474113f9d2f92c98d6c774e4c9cb702b2b21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, release=754, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, build-date=2022-05-03T08:36:31.336870, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-type=git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., io.buildah.version=1.19.8, RELEASE=HEAD, com.redhat.component=centos-stream-container, version=8, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public) 2024-09-13T13:06:57.835 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:57 smithi046 bash[63508]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6 2024-09-13T13:06:58.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:57 smithi046 podman[63525]: 2024-09-13 13:06:57.834598377 +0000 UTC m=+0.624300512 container remove 6f862726cfc5a1e2b390c5bbbebf474113f9d2f92c98d6c774e4c9cb702b2b21 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, RELEASE=HEAD, com.redhat.component=centos-stream-container, name=centos-stream, ceph=True, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-09-13T13:06:58.597 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:58 smithi046 podman[63611]: 2024-09-13 13:06:58.226989876 +0000 UTC m=+0.018068437 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:06:58.597 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:58 smithi046 podman[63611]: 2024-09-13 13:06:58.344321374 +0000 UTC m=+0.135399910 container create 4ef9cfce5721e82280c8b4b2403fb6baae19a3fcbceb1d9e7da461347c08ceca (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T13:06:58.598 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:58 smithi046 podman[63611]: 2024-09-13 13:06:58.597064955 +0000 UTC m=+0.388143492 container init 4ef9cfce5721e82280c8b4b2403fb6baae19a3fcbceb1d9e7da461347c08ceca (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-13T13:06:58.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 236283 8.84 MB/sec execute 543 sec latency 39.270 ms 2024-09-13T13:06:58.764 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:58 smithi042 ceph-mon[63970]: pgmap v265: 97 pgs: 7 peering, 8 stale+active+clean, 82 active+clean; 363 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 392 op/s 2024-09-13T13:06:58.764 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:58 smithi042 ceph-mon[63970]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T13:06:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:58 smithi046 ceph-mon[55972]: pgmap v265: 97 pgs: 7 peering, 8 stale+active+clean, 82 active+clean; 363 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 6.8 KiB/s rd, 4.4 MiB/s wr, 392 op/s 2024-09-13T13:06:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:58 smithi046 ceph-mon[55972]: osdmap e149: 8 total, 7 up, 8 in 2024-09-13T13:06:58.867 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:58 smithi046 podman[63611]: 2024-09-13 13:06:58.601105067 +0000 UTC m=+0.392183611 container start 4ef9cfce5721e82280c8b4b2403fb6baae19a3fcbceb1d9e7da461347c08ceca (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-13T13:06:58.867 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:58 smithi046 podman[63611]: 2024-09-13 13:06:58.697300011 +0000 UTC m=+0.488378558 container attach 4ef9cfce5721e82280c8b4b2403fb6baae19a3fcbceb1d9e7da461347c08ceca (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T13:06:58.868 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:58 smithi046 podman[63611]: 2024-09-13 13:06:58.825078584 +0000 UTC m=+0.616157124 container died 4ef9cfce5721e82280c8b4b2403fb6baae19a3fcbceb1d9e7da461347c08ceca (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:06:59.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 236546 8.83 MB/sec execute 544 sec latency 63.702 ms 2024-09-13T13:06:59.739 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:59 smithi046 podman[63635]: 2024-09-13 13:06:59.452404146 +0000 UTC m=+0.620914768 container remove 4ef9cfce5721e82280c8b4b2403fb6baae19a3fcbceb1d9e7da461347c08ceca (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:06:59.739 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:59 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.6.service: Deactivated successfully. 2024-09-13T13:06:59.739 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:59 smithi046 systemd[1]: Stopped Ceph osd.6 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:06:59.739 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:59 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.6.service: Consumed 4min 9.114s CPU time. 2024-09-13T13:06:59.993 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:59 smithi046 systemd[1]: Starting Ceph osd.6 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:06:59.993 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:06:59.993 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:06:59.993 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:06:59 smithi046 ceph-mon[55972]: pgmap v267: 97 pgs: 7 active+undersized, 7 peering, 2 stale+active+clean, 16 active+undersized+degraded, 65 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 322 op/s; 120/1596 objects degraded (7.519%) 2024-09-13T13:07:00.069 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:00.069 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:00.069 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:06:59 smithi042 ceph-mon[63970]: pgmap v267: 97 pgs: 7 active+undersized, 7 peering, 2 stale+active+clean, 16 active+undersized+degraded, 65 active+clean; 357 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 4.4 KiB/s rd, 4.2 MiB/s wr, 322 op/s; 120/1596 objects degraded (7.519%) 2024-09-13T13:07:00.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:06:59 smithi046 podman[63729]: 2024-09-13 13:06:59.897625626 +0000 UTC m=+0.024208097 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:07:00.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 podman[63729]: 2024-09-13 13:07:00.028608324 +0000 UTC m=+0.155190785 container create 51aee9488e1b389469c82f00d98cac657fd0c456e35d2497024f8fe3c1c77c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T13:07:00.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 236849 8.83 MB/sec execute 545 sec latency 42.071 ms 2024-09-13T13:07:00.849 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 podman[63729]: 2024-09-13 13:07:00.448695613 +0000 UTC m=+0.575278082 container init 51aee9488e1b389469c82f00d98cac657fd0c456e35d2497024f8fe3c1c77c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True) 2024-09-13T13:07:00.849 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 podman[63729]: 2024-09-13 13:07:00.453975987 +0000 UTC m=+0.580558445 container start 51aee9488e1b389469c82f00d98cac657fd0c456e35d2497024f8fe3c1c77c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:07:00.849 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 podman[63729]: 2024-09-13 13:07:00.556877088 +0000 UTC m=+0.683459548 container attach 51aee9488e1b389469c82f00d98cac657fd0c456e35d2497024f8fe3c1c77c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T13:07:00.850 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:00.850 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 bash[63729]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:00.850 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:00.850 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:00 smithi046 bash[63729]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:01.092 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:00 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 120/1596 objects degraded (7.519%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:00 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 120/1596 objects degraded (7.519%), 16 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:01.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 237107 8.83 MB/sec execute 546 sec latency 51.120 ms 2024-09-13T13:07:02.073 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:07:02.110 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:01 smithi042 ceph-mon[63970]: pgmap v268: 97 pgs: 8 active+undersized, 7 peering, 1 stale+active+clean, 19 active+undersized+degraded, 62 active+clean; 359 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.1 MiB/s wr, 302 op/s; 140/1584 objects degraded (8.838%) 2024-09-13T13:07:02.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:01 smithi046 ceph-mon[55972]: pgmap v268: 97 pgs: 8 active+undersized, 7 peering, 1 stale+active+clean, 19 active+undersized+degraded, 62 active+clean; 359 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.1 MiB/s wr, 302 op/s; 140/1584 objects degraded (8.838%) 2024-09-13T13:07:02.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 237397 8.82 MB/sec execute 547 sec latency 69.211 ms 2024-09-13T13:07:02.886 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:02 smithi042 ceph-mon[63970]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:02.886 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:02 smithi042 ceph-mon[63970]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:02.953 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:07:02.953 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (8m) 112s ago 20m 19.8M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:07:02.953 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (5m) 112s ago 20m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:07:02.953 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (5m) 32s ago 17m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:07:02.953 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (8m) 112s ago 17m 45.1M - dad864ee21e9 7144014c89b8 2024-09-13T13:07:02.953 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (11m) 112s ago 11m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (11m) 32s ago 11m 16.4M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (7m) 112s ago 21m 519M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (6m) 32s ago 17m 456M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (6m) 112s ago 22m 56.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (6m) 32s ago 17m 47.8M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (11m) 112s ago 11m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (11m) 32s ago 11m 90.9M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (8m) 112s ago 17m 10.7M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (7m) 32s ago 16m 12.2M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (4m) 112s ago 16m 335M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (3m) 112s ago 15m 254M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (2m) 112s ago 15m 179M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:07:02.954 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (2m) 112s ago 14m 11.9M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:07:02.955 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (75s) 32s ago 14m 270M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:07:02.955 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (37s) 32s ago 14m 12.1M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:07:02.955 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (13m) 32s ago 13m 469M 2705M 17.2.0 e1d6a67b021e 6f862726cfc5 2024-09-13T13:07:02.955 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (13m) 32s ago 13m 736M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:07:02.955 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (6m) 112s ago 16m 55.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:07:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:02 smithi046 ceph-mon[55972]: from='client.34408 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:02 smithi046 ceph-mon[55972]: from='client.34412 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:03.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T13:07:03.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T13:07:03.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T13:07:03.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T13:07:03.575 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 6 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:03.576 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 10 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:07:03.577 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:07:03.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T13:07:03.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T13:07:03.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T13:07:03.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T13:07:03.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T13:07:03.618 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-13T13:07:03.618 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T13:07:03.618 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-13T13:07:03.618 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate[63743]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T13:07:03.618 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 bash[63729]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-13T13:07:03.618 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:03 smithi046 podman[63729]: 2024-09-13 13:07:03.37529183 +0000 UTC m=+3.501874291 container died 51aee9488e1b389469c82f00d98cac657fd0c456e35d2497024f8fe3c1c77c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:07:03.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 237722 8.83 MB/sec execute 548 sec latency 46.779 ms 2024-09-13T13:07:03.919 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:03 smithi042 ceph-mon[63970]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:03.919 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:03 smithi042 ceph-mon[63970]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 423 op/s; 179/1551 objects degraded (11.541%) 2024-09-13T13:07:03.919 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:03 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1279352978' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:07:04.049 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:07:04.050 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:07:04.050 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:07:04.050 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "12/23 daemons upgraded", 2024-09-13T13:07:04.050 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:07:04.050 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:07:04.050 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:07:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-mon[55972]: from='client.34416 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-mon[55972]: pgmap v269: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 8.2 KiB/s rd, 3.7 MiB/s wr, 423 op/s; 179/1551 objects degraded (11.541%) 2024-09-13T13:07:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:03 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1279352978' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:04.530 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 podman[63902]: 2024-09-13 13:07:04.16267414 +0000 UTC m=+0.776411273 container remove 51aee9488e1b389469c82f00d98cac657fd0c456e35d2497024f8fe3c1c77c6d (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T13:07:04.530 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 podman[63958]: 2024-09-13 13:07:04.288160452 +0000 UTC m=+0.019371619 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:07:04.530 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 podman[63958]: 2024-09-13 13:07:04.405436653 +0000 UTC m=+0.136647794 container create 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:07:04.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 237981 8.82 MB/sec execute 549 sec latency 43.296 ms 2024-09-13T13:07:04.720 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down; Degraded data redundancy: 179/1551 objects degraded (11.541%), 25 pgs degraded 2024-09-13T13:07:04.720 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:07:04.720 INFO:teuthology.orchestra.run.smithi042.stdout: osd.6 (root=default,host=smithi046) is down 2024-09-13T13:07:04.721 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 179/1551 objects degraded (11.541%), 25 pgs degraded 2024-09-13T13:07:04.721 INFO:teuthology.orchestra.run.smithi042.stdout: pg 1.0 is active+undersized+degraded, acting [7,0] 2024-09-13T13:07:04.721 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-09-13T13:07:04.721 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.6 is active+undersized+degraded, acting [1,4] 2024-09-13T13:07:04.721 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.7 is active+undersized+degraded, acting [4,2] 2024-09-13T13:07:04.722 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.11 is active+undersized+degraded, acting [4,1] 2024-09-13T13:07:04.722 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.14 is active+undersized+degraded, acting [3,5] 2024-09-13T13:07:04.722 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.16 is active+undersized+degraded, acting [5,2] 2024-09-13T13:07:04.722 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.17 is active+undersized+degraded, acting [5,2] 2024-09-13T13:07:04.722 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.19 is active+undersized+degraded, acting [3,0] 2024-09-13T13:07:04.722 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1a is active+undersized+degraded, acting [4,7] 2024-09-13T13:07:04.723 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1b is active+undersized+degraded, acting [3,7] 2024-09-13T13:07:04.723 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1d is active+undersized+degraded, acting [7,0] 2024-09-13T13:07:04.723 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.0 is active+undersized+degraded, acting [1,2] 2024-09-13T13:07:04.723 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.2 is active+undersized+degraded, acting [3,5] 2024-09-13T13:07:04.723 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.3 is active+undersized+degraded, acting [4,0] 2024-09-13T13:07:04.723 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.a is active+undersized+degraded, acting [4,1] 2024-09-13T13:07:04.724 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.c is active+undersized+degraded, acting [5,3] 2024-09-13T13:07:04.724 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.d is active+undersized+degraded, acting [7,5] 2024-09-13T13:07:04.724 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.10 is active+undersized+degraded, acting [0,5] 2024-09-13T13:07:04.724 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.11 is active+undersized+degraded, acting [7,4] 2024-09-13T13:07:04.724 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-09-13T13:07:04.724 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-09-13T13:07:04.725 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1e is active+undersized+degraded, acting [3,2] 2024-09-13T13:07:04.725 INFO:teuthology.orchestra.run.smithi042.stdout: pg 4.5 is active+undersized+degraded, acting [4,7] 2024-09-13T13:07:04.725 INFO:teuthology.orchestra.run.smithi042.stdout: pg 4.a is active+undersized+degraded, acting [1,7] 2024-09-13T13:07:04.970 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:04 smithi046 ceph-mon[55972]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:04.971 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:04 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2838344466' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:07:04.971 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:04.971 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:04.971 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:04 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:04.972 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 podman[63958]: 2024-09-13 13:07:04.675329492 +0000 UTC m=+0.406540641 container init 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS) 2024-09-13T13:07:04.972 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 podman[63958]: 2024-09-13 13:07:04.678891613 +0000 UTC m=+0.410102764 container start 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:07:04.972 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 ceph-osd[63977]: -- 172.21.15.46:0/2097414555 <== mon.1 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55bb4dd421e0 con 0x55bb4dd34000 2024-09-13T13:07:04.972 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 bash[63958]: 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 2024-09-13T13:07:04.972 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:04 smithi046 systemd[1]: Started Ceph osd.6 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:07:04.973 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:04 smithi042 ceph-mon[63970]: from='client.34424 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:04.973 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:04 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2838344466' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:07:04.973 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:04.974 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:04.974 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:04 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:05.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 238355 8.84 MB/sec execute 550 sec latency 72.378 ms 2024-09-13T13:07:05.987 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:05 smithi042 ceph-mon[63970]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.2 MiB/s wr, 339 op/s; 176/1566 objects degraded (11.239%) 2024-09-13T13:07:05.987 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:05 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 179/1551 objects degraded (11.541%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:05 smithi046 ceph-mon[55972]: pgmap v270: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.5 KiB/s rd, 3.2 MiB/s wr, 339 op/s; 176/1566 objects degraded (11.239%) 2024-09-13T13:07:06.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:05 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 179/1551 objects degraded (11.541%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:06.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 238641 8.83 MB/sec execute 551 sec latency 43.647 ms 2024-09-13T13:07:06.867 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:07:06.570+0000 7feed4c65640 -1 Falling back to public interface 2024-09-13T13:07:07.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 239062 8.84 MB/sec execute 552 sec latency 29.123 ms 2024-09-13T13:07:08.517 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:08 smithi042 ceph-mon[63970]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 360 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 438 op/s; 182/1653 objects degraded (11.010%) 2024-09-13T13:07:08.648 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:08 smithi046 ceph-mon[55972]: pgmap v271: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 360 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 3.9 MiB/s wr, 438 op/s; 182/1653 objects degraded (11.010%) 2024-09-13T13:07:08.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 239475 8.83 MB/sec execute 553 sec latency 38.571 ms 2024-09-13T13:07:09.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 239742 8.83 MB/sec execute 554 sec latency 57.968 ms 2024-09-13T13:07:09.802 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:09 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.802 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:09 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.802 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:09 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.802 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:09 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.830 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:09 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.830 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:09 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.830 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:09 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:09.830 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:09 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:10.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 240044 8.82 MB/sec execute 555 sec latency 63.919 ms 2024-09-13T13:07:10.824 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:10 smithi042 ceph-mon[63970]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 365 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.6 MiB/s wr, 433 op/s; 194/1707 objects degraded (11.365%) 2024-09-13T13:07:10.824 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:10 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 194/1707 objects degraded (11.365%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:10 smithi046 ceph-mon[55972]: pgmap v272: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 365 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 4.6 KiB/s rd, 4.6 MiB/s wr, 433 op/s; 194/1707 objects degraded (11.365%) 2024-09-13T13:07:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:10 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 194/1707 objects degraded (11.365%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:11.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 240455 8.83 MB/sec execute 556 sec latency 76.528 ms 2024-09-13T13:07:12.596 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:12 smithi042 ceph-mon[63970]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 424 op/s; 192/1683 objects degraded (11.408%) 2024-09-13T13:07:12.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 240743 8.83 MB/sec execute 557 sec latency 65.871 ms 2024-09-13T13:07:12.746 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:12 smithi046 ceph-mon[55972]: pgmap v273: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.9 KiB/s rd, 4.1 MiB/s wr, 424 op/s; 192/1683 objects degraded (11.408%) 2024-09-13T13:07:13.106 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:07:12.822+0000 7feed4c65640 -1 osd.6 0 read_superblock omap replica is missing. 2024-09-13T13:07:13.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 241118 8.83 MB/sec execute 558 sec latency 53.215 ms 2024-09-13T13:07:14.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:07:13.988+0000 7feed4c65640 -1 osd.6 147 log_to_monitors true 2024-09-13T13:07:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:14.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 177/1593 objects degraded (11.111%) 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='osd.6 [v2:172.21.15.46:6816/2734277987,v1:172.21.15.46:6817/2734277987]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T13:07:14.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-mon[55972]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T13:07:14.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:14.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:07:14.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.383 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: pgmap v274: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 351 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.9 MiB/s wr, 474 op/s; 177/1593 objects degraded (11.111%) 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='osd.6 [v2:172.21.15.46:6816/2734277987,v1:172.21.15.46:6817/2734277987]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T13:07:14.384 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:14 smithi042 ceph-mon[63970]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-13T13:07:14.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 241447 8.83 MB/sec execute 559 sec latency 34.317 ms 2024-09-13T13:07:15.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:07:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:07:14.854+0000 7feecca0f640 -1 osd.6 147 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:07:15.407 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:15 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T13:07:15.407 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:15 smithi042 ceph-mon[63970]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-13T13:07:15.407 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:15 smithi042 ceph-mon[63970]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T13:07:15.407 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:15 smithi042 ceph-mon[63970]: from='osd.6 [v2:172.21.15.46:6816/2734277987,v1:172.21.15.46:6817/2734277987]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:15.408 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:15 smithi042 ceph-mon[63970]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T13:07:15.408 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:15 smithi042 ceph-mon[63970]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:15 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T13:07:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:15 smithi046 ceph-mon[55972]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-13T13:07:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:15 smithi046 ceph-mon[55972]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T13:07:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:15 smithi046 ceph-mon[55972]: from='osd.6 [v2:172.21.15.46:6816/2734277987,v1:172.21.15.46:6817/2734277987]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:15 smithi046 ceph-mon[55972]: osdmap e150: 8 total, 7 up, 8 in 2024-09-13T13:07:15.618 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:15 smithi046 ceph-mon[55972]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:15.683 INFO:teuthology.orchestra.run.smithi042.stdout: 5 241758 8.84 MB/sec execute 560 sec latency 64.885 ms 2024-09-13T13:07:16.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:16 smithi042 ceph-mon[63970]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 458 op/s; 176/1563 objects degraded (11.260%) 2024-09-13T13:07:16.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:16 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 177/1593 objects degraded (11.111%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:16.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:16 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:07:16.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:16 smithi042 ceph-mon[63970]: osd.6 [v2:172.21.15.46:6816/2734277987,v1:172.21.15.46:6817/2734277987] boot 2024-09-13T13:07:16.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:16 smithi042 ceph-mon[63970]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T13:07:16.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:16 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T13:07:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:16 smithi046 ceph-mon[55972]: pgmap v276: 97 pgs: 10 active+undersized, 25 active+undersized+degraded, 62 active+clean; 350 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.9 MiB/s wr, 458 op/s; 176/1563 objects degraded (11.260%) 2024-09-13T13:07:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:16 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 177/1593 objects degraded (11.111%), 25 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:16 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:07:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:16 smithi046 ceph-mon[55972]: osd.6 [v2:172.21.15.46:6816/2734277987,v1:172.21.15.46:6817/2734277987] boot 2024-09-13T13:07:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:16 smithi046 ceph-mon[55972]: osdmap e151: 8 total, 8 up, 8 in 2024-09-13T13:07:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:16 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-13T13:07:16.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 241877 8.83 MB/sec execute 561 sec latency 835.304 ms 2024-09-13T13:07:17.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 241961 8.82 MB/sec execute 562 sec latency 1770.245 ms 2024-09-13T13:07:17.986 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:17 smithi042 ceph-mon[63970]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T13:07:17.986 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:17 smithi042 ceph-mon[63970]: pgmap v279: 97 pgs: 8 peering, 8 active+undersized, 19 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.0 MiB/s wr, 562 op/s; 132/1644 objects degraded (8.029%) 2024-09-13T13:07:18.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:17 smithi046 ceph-mon[55972]: osdmap e152: 8 total, 8 up, 8 in 2024-09-13T13:07:18.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:17 smithi046 ceph-mon[55972]: pgmap v279: 97 pgs: 8 peering, 8 active+undersized, 19 active+undersized+degraded, 62 active+clean; 355 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 5.0 MiB/s wr, 562 op/s; 132/1644 objects degraded (8.029%) 2024-09-13T13:07:18.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 242203 8.81 MB/sec execute 563 sec latency 2103.050 ms 2024-09-13T13:07:19.008 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:18 smithi042 ceph-mon[63970]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T13:07:19.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:18 smithi046 ceph-mon[55972]: osdmap e153: 8 total, 8 up, 8 in 2024-09-13T13:07:19.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 242492 8.81 MB/sec execute 564 sec latency 44.124 ms 2024-09-13T13:07:20.040 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:19 smithi042 ceph-mon[63970]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T13:07:20.040 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:19 smithi042 ceph-mon[63970]: pgmap v282: 97 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 8 peering, 1 active+undersized, 9 active+undersized+degraded, 77 active+clean; 357 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.3 MiB/s wr, 520 op/s; 73/1626 objects degraded (4.490%); 24 KiB/s, 18 objects/s recovering 2024-09-13T13:07:20.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:19 smithi046 ceph-mon[55972]: osdmap e154: 8 total, 8 up, 8 in 2024-09-13T13:07:20.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:19 smithi046 ceph-mon[55972]: pgmap v282: 97 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 8 peering, 1 active+undersized, 9 active+undersized+degraded, 77 active+clean; 357 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 5.3 MiB/s wr, 520 op/s; 73/1626 objects degraded (4.490%); 24 KiB/s, 18 objects/s recovering 2024-09-13T13:07:20.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 242767 8.82 MB/sec execute 565 sec latency 833.634 ms 2024-09-13T13:07:21.059 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:20 smithi042 ceph-mon[63970]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T13:07:21.059 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:20 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 73/1626 objects degraded (4.490%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:20 smithi046 ceph-mon[55972]: osdmap e155: 8 total, 8 up, 8 in 2024-09-13T13:07:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:20 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 73/1626 objects degraded (4.490%), 10 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:21.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 243041 8.80 MB/sec execute 566 sec latency 1038.867 ms 2024-09-13T13:07:22.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:21 smithi042 ceph-mon[63970]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T13:07:22.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:21 smithi042 ceph-mon[63970]: pgmap v285: 97 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 6 peering, 1 active+undersized+degraded, 88 active+clean; 361 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.6 MiB/s wr, 322 op/s; 40/1635 objects degraded (2.446%); 270 KiB/s, 69 keys/s, 27 objects/s recovering 2024-09-13T13:07:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:21 smithi046 ceph-mon[55972]: osdmap e156: 8 total, 8 up, 8 in 2024-09-13T13:07:22.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:21 smithi046 ceph-mon[55972]: pgmap v285: 97 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 6 peering, 1 active+undersized+degraded, 88 active+clean; 361 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.2 KiB/s rd, 3.6 MiB/s wr, 322 op/s; 40/1635 objects degraded (2.446%); 270 KiB/s, 69 keys/s, 27 objects/s recovering 2024-09-13T13:07:22.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 243315 8.80 MB/sec execute 567 sec latency 52.994 ms 2024-09-13T13:07:23.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 243540 8.79 MB/sec execute 568 sec latency 47.847 ms 2024-09-13T13:07:24.645 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:24 smithi042 ceph-mon[63970]: pgmap v286: 97 pgs: 1 peering, 1 active+undersized+degraded, 95 active+clean; 359 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 296 op/s; 13/1611 objects degraded (0.807%); 530 KiB/s, 50 keys/s, 33 objects/s recovering 2024-09-13T13:07:24.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 243838 8.79 MB/sec execute 569 sec latency 70.535 ms 2024-09-13T13:07:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:24 smithi046 ceph-mon[55972]: pgmap v286: 97 pgs: 1 peering, 1 active+undersized+degraded, 95 active+clean; 359 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.7 KiB/s rd, 3.8 MiB/s wr, 296 op/s; 13/1611 objects degraded (0.807%); 530 KiB/s, 50 keys/s, 33 objects/s recovering 2024-09-13T13:07:25.517 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:25 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1611 objects degraded (0.807%), 1 pg degraded) 2024-09-13T13:07:25.517 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:25 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:07:25.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 244179 8.79 MB/sec execute 570 sec latency 56.108 ms 2024-09-13T13:07:25.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:25 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/1611 objects degraded (0.807%), 1 pg degraded) 2024-09-13T13:07:25.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:25 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:07:26.463 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:26 smithi042 ceph-mon[63970]: pgmap v287: 97 pgs: 97 active+clean; 359 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 453 op/s; 1.1 MiB/s, 42 keys/s, 28 objects/s recovering 2024-09-13T13:07:26.684 INFO:teuthology.orchestra.run.smithi042.stdout: 5 244540 8.79 MB/sec execute 571 sec latency 63.668 ms 2024-09-13T13:07:26.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:26 smithi046 ceph-mon[55972]: pgmap v287: 97 pgs: 97 active+clean; 359 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 453 op/s; 1.1 MiB/s, 42 keys/s, 28 objects/s recovering 2024-09-13T13:07:27.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 244975 8.80 MB/sec execute 572 sec latency 43.306 ms 2024-09-13T13:07:28.480 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:28 smithi042 ceph-mon[63970]: pgmap v288: 97 pgs: 97 active+clean; 355 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.6 MiB/s wr, 400 op/s; 875 KiB/s, 34 keys/s, 18 objects/s recovering 2024-09-13T13:07:28.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 245240 8.80 MB/sec execute 573 sec latency 56.066 ms 2024-09-13T13:07:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:28 smithi046 ceph-mon[55972]: pgmap v288: 97 pgs: 97 active+clean; 355 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.0 KiB/s rd, 3.6 MiB/s wr, 400 op/s; 875 KiB/s, 34 keys/s, 18 objects/s recovering 2024-09-13T13:07:29.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 245616 8.80 MB/sec execute 574 sec latency 47.550 ms 2024-09-13T13:07:30.008 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:30.008 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:30.008 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T13:07:30.008 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:29 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T13:07:30.008 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:29 smithi042 ceph-mon[63970]: Upgrade: osd.7 is safe to restart 2024-09-13T13:07:30.009 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:29 smithi042 ceph-mon[63970]: pgmap v289: 97 pgs: 97 active+clean; 356 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.3 MiB/s wr, 443 op/s; 737 KiB/s, 29 keys/s, 15 objects/s recovering 2024-09-13T13:07:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-13T13:07:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:29 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-13T13:07:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:29 smithi046 ceph-mon[55972]: Upgrade: osd.7 is safe to restart 2024-09-13T13:07:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:29 smithi046 ceph-mon[55972]: pgmap v289: 97 pgs: 97 active+clean; 356 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.8 KiB/s rd, 4.3 MiB/s wr, 443 op/s; 737 KiB/s, 29 keys/s, 15 objects/s recovering 2024-09-13T13:07:30.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 245886 8.80 MB/sec execute 575 sec latency 46.025 ms 2024-09-13T13:07:31.289 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:31 smithi042 ceph-mon[63970]: Upgrade: Updating osd.7 2024-09-13T13:07:31.290 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:31.290 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T13:07:31.290 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:31 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:31.290 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:31 smithi042 ceph-mon[63970]: Deploying daemon osd.7 on smithi046 2024-09-13T13:07:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:31 smithi046 ceph-mon[55972]: Upgrade: Updating osd.7 2024-09-13T13:07:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-13T13:07:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:31 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:31.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:31 smithi046 ceph-mon[55972]: Deploying daemon osd.7 on smithi046 2024-09-13T13:07:31.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 246175 8.81 MB/sec execute 576 sec latency 51.275 ms 2024-09-13T13:07:32.493 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:32 smithi046 ceph-mon[55972]: pgmap v290: 97 pgs: 97 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 460 op/s; 646 KiB/s, 26 keys/s, 12 objects/s recovering 2024-09-13T13:07:32.548 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:32 smithi042 ceph-mon[63970]: pgmap v290: 97 pgs: 97 active+clean; 353 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 KiB/s rd, 4.2 MiB/s wr, 460 op/s; 646 KiB/s, 26 keys/s, 12 objects/s recovering 2024-09-13T13:07:32.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 246446 8.80 MB/sec execute 577 sec latency 58.127 ms 2024-09-13T13:07:33.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:33 smithi046 systemd[1]: Stopping Ceph osd.7 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:07:33.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 246692 8.79 MB/sec execute 578 sec latency 55.989 ms 2024-09-13T13:07:33.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:33 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[45508]: 2024-09-13T13:07:33.564+0000 7f43eeead700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:07:33.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:33 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[45508]: 2024-09-13T13:07:33.564+0000 7f43eeead700 -1 osd.7 156 *** Got signal Terminated *** 2024-09-13T13:07:33.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:33 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[45508]: 2024-09-13T13:07:33.564+0000 7f43eeead700 -1 osd.7 156 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:07:34.461 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:34 smithi042 ceph-mon[63970]: pgmap v291: 97 pgs: 97 active+clean; 355 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s; 501 KiB/s, 9 objects/s recovering 2024-09-13T13:07:34.461 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:34 smithi042 ceph-mon[63970]: osd.7 marked itself down and dead 2024-09-13T13:07:34.461 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:34 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:34.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 246902 8.78 MB/sec execute 579 sec latency 235.696 ms 2024-09-13T13:07:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:34 smithi046 ceph-mon[55972]: pgmap v291: 97 pgs: 97 active+clean; 355 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 KiB/s rd, 4.6 MiB/s wr, 441 op/s; 501 KiB/s, 9 objects/s recovering 2024-09-13T13:07:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:34 smithi046 ceph-mon[55972]: osd.7 marked itself down and dead 2024-09-13T13:07:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:34 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:35.313 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:35 smithi046 podman[65540]: 2024-09-13 13:07:35.022929796 +0000 UTC m=+1.649461583 container died d4b77890db2635351fbf727f2312f3b8e76166011bbc5aba56acf1502037d8d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, io.openshift.expose-services=, vcs-type=git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc.) 2024-09-13T13:07:35.336 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:07:35.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:35 smithi042 ceph-mon[63970]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:07:35.600 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:35 smithi042 ceph-mon[63970]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T13:07:35.601 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:35 smithi042 ceph-mon[63970]: osdmap e157: 8 total, 7 up, 8 in 2024-09-13T13:07:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:35 smithi046 ceph-mon[55972]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T13:07:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:35 smithi046 ceph-mon[55972]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-13T13:07:35.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:35 smithi046 ceph-mon[55972]: osdmap e157: 8 total, 7 up, 8 in 2024-09-13T13:07:35.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:35 smithi046 podman[65540]: 2024-09-13 13:07:35.411957594 +0000 UTC m=+2.038489366 container cleanup d4b77890db2635351fbf727f2312f3b8e76166011bbc5aba56acf1502037d8d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, release=754, io.openshift.expose-services=, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, version=8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, distribution-scope=public, vendor=Red Hat, Inc., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8) 2024-09-13T13:07:35.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:35 smithi046 bash[65540]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7 2024-09-13T13:07:35.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 247020 8.77 MB/sec execute 580 sec latency 1027.469 ms 2024-09-13T13:07:36.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65575]: 2024-09-13 13:07:36.1053894 +0000 UTC m=+1.081726035 container remove d4b77890db2635351fbf727f2312f3b8e76166011bbc5aba56acf1502037d8d1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, RELEASE=HEAD, release=754, vendor=Red Hat, Inc., distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base centos centos-stream, version=8, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , name=centos-stream) 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (9m) 2m ago 21m 19.8M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (5m) 2m ago 21m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (5m) 27s ago 17m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (8m) 2m ago 18m 45.1M - dad864ee21e9 7144014c89b8 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (12m) 2m ago 12m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (12m) 27s ago 12m 16.5M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (8m) 2m ago 22m 519M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (7m) 27s ago 17m 456M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (7m) 2m ago 22m 56.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (6m) 27s ago 17m 48.8M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:07:36.491 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (11m) 2m ago 11m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (11m) 27s ago 11m 90.9M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (8m) 2m ago 18m 10.7M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (8m) 27s ago 17m 12.1M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (5m) 2m ago 16m 335M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (4m) 2m ago 16m 254M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (3m) 2m ago 15m 179M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (2m) 2m ago 15m 11.9M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (109s) 27s ago 15m 305M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (70s) 27s ago 14m 251M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (32s) 27s ago 14m 12.1M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 9d85dde44d01 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (13m) 27s ago 13m 785M 2705M 17.2.0 e1d6a67b021e d4b77890db26 2024-09-13T13:07:36.492 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (7m) 2m ago 17m 55.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:07:36.652 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:36 smithi042 ceph-mon[63970]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:36.653 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:36 smithi042 ceph-mon[63970]: pgmap v293: 97 pgs: 4 peering, 12 stale+active+clean, 81 active+clean; 355 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 364 op/s 2024-09-13T13:07:36.653 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:36 smithi042 ceph-mon[63970]: osdmap e158: 8 total, 7 up, 8 in 2024-09-13T13:07:36.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 247234 8.77 MB/sec execute 581 sec latency 56.690 ms 2024-09-13T13:07:36.793 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:36 smithi046 ceph-mon[55972]: from='client.34432 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:36.793 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:36 smithi046 ceph-mon[55972]: pgmap v293: 97 pgs: 4 peering, 12 stale+active+clean, 81 active+clean; 355 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.6 KiB/s rd, 4.3 MiB/s wr, 364 op/s 2024-09-13T13:07:36.793 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:36 smithi046 ceph-mon[55972]: osdmap e158: 8 total, 7 up, 8 in 2024-09-13T13:07:36.793 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65663]: 2024-09-13 13:07:36.4115113 +0000 UTC m=+0.016346965 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:07:36.793 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65663]: 2024-09-13 13:07:36.523003989 +0000 UTC m=+0.127839641 container create 40932ff9360ced5f5457d6c21af7aaeb4469b71abc11ed2e2cec6d0f08f00f21 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T13:07:36.793 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65663]: 2024-09-13 13:07:36.792537411 +0000 UTC m=+0.397373064 container init 40932ff9360ced5f5457d6c21af7aaeb4469b71abc11ed2e2cec6d0f08f00f21 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:07:37.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65663]: 2024-09-13 13:07:36.796307822 +0000 UTC m=+0.401143475 container start 40932ff9360ced5f5457d6c21af7aaeb4469b71abc11ed2e2cec6d0f08f00f21 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, ceph=True) 2024-09-13T13:07:37.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65663]: 2024-09-13 13:07:36.884525855 +0000 UTC m=+0.489361525 container attach 40932ff9360ced5f5457d6c21af7aaeb4469b71abc11ed2e2cec6d0f08f00f21 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2) 2024-09-13T13:07:37.118 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:36 smithi046 podman[65663]: 2024-09-13 13:07:36.999487813 +0000 UTC m=+0.604323476 container died 40932ff9360ced5f5457d6c21af7aaeb4469b71abc11ed2e2cec6d0f08f00f21 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:07:37.178 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 7 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 11 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:07:37.179 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:07:37.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 247431 8.76 MB/sec execute 582 sec latency 85.564 ms 2024-09-13T13:07:37.690 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:37 smithi042 ceph-mon[63970]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:37.690 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:37 smithi042 ceph-mon[63970]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:37.690 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:37 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/3743876998' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:37.767 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:07:37.767 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:07:37.767 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:07:37.767 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:07:37.767 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:07:37.767 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "13/23 daemons upgraded", 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading osd daemons", 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:07:37.768 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:07:37.772 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:37 smithi046 ceph-mon[55972]: from='client.34436 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:37.772 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:37 smithi046 ceph-mon[55972]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:37.772 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:37 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/3743876998' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:38.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 podman[65685]: 2024-09-13 13:07:38.072627741 +0000 UTC m=+1.066869772 container remove 40932ff9360ced5f5457d6c21af7aaeb4469b71abc11ed2e2cec6d0f08f00f21 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T13:07:38.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.7.service: Deactivated successfully. 2024-09-13T13:07:38.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 systemd[1]: Stopped Ceph osd.7 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:07:38.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 systemd[1]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.7.service: Consumed 6min 52.906s CPU time. 2024-09-13T13:07:38.461 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_WARN 1 osds down; all OSDs are running squid or later but require_osd_release < squid; Degraded data redundancy: 106/1599 objects degraded (6.629%), 17 pgs degraded 2024-09-13T13:07:38.461 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_DOWN: 1 osds down 2024-09-13T13:07:38.461 INFO:teuthology.orchestra.run.smithi042.stdout: osd.7 (root=default,host=smithi046) is down 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] OSD_UPGRADE_FINISHED: all OSDs are running squid or later but require_osd_release < squid 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: all OSDs are running squid or later but require_osd_release < squid 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout:[WRN] PG_DEGRADED: Degraded data redundancy: 106/1599 objects degraded (6.629%), 17 pgs degraded 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 1.0 is active+undersized+degraded, acting [0,6] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.0 is active+undersized+degraded, acting [1,0] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.3 is active+undersized+degraded, acting [5,2] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.4 is active+undersized+degraded, acting [1,0] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.5 is active+undersized+degraded, acting [0,4] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.8 is active+undersized+degraded, acting [0,1] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.9 is active+undersized+degraded, acting [1,3] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.a is active+undersized+degraded, acting [1,3] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.e is active+undersized+degraded, acting [2,3] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.12 is active+undersized+degraded, acting [5,3] 2024-09-13T13:07:38.462 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.18 is active+undersized+degraded, acting [5,4] 2024-09-13T13:07:38.463 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1c is active+undersized+degraded, acting [5,2] 2024-09-13T13:07:38.463 INFO:teuthology.orchestra.run.smithi042.stdout: pg 2.1f is active+undersized+degraded, acting [0,4] 2024-09-13T13:07:38.463 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.d is active+undersized+degraded, acting [5,6] 2024-09-13T13:07:38.463 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.12 is active+undersized+degraded, acting [0,3] 2024-09-13T13:07:38.463 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.16 is active+undersized+degraded, acting [5,1] 2024-09-13T13:07:38.463 INFO:teuthology.orchestra.run.smithi042.stdout: pg 3.1b is active+undersized+degraded, acting [0,4] 2024-09-13T13:07:38.636 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:38 smithi046 ceph-mon[55972]: pgmap v295: 97 pgs: 5 active+undersized, 4 peering, 6 stale+active+clean, 17 active+undersized+degraded, 65 active+clean; 354 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 454 op/s; 106/1599 objects degraded (6.629%) 2024-09-13T13:07:38.636 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:38 smithi046 ceph-mon[55972]: Health check failed: Degraded data redundancy: 106/1599 objects degraded (6.629%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:38.637 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:38 smithi046 ceph-mon[55972]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:38.637 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:38 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1728395737' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:07:38.637 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 systemd[1]: Starting Ceph osd.7 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:07:38.637 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 podman[65786]: 2024-09-13 13:07:38.515564795 +0000 UTC m=+0.016844344 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:07:38.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 247715 8.76 MB/sec execute 583 sec latency 145.320 ms 2024-09-13T13:07:38.724 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:38 smithi042 ceph-mon[63970]: pgmap v295: 97 pgs: 5 active+undersized, 4 peering, 6 stale+active+clean, 17 active+undersized+degraded, 65 active+clean; 354 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 4.2 MiB/s wr, 454 op/s; 106/1599 objects degraded (6.629%) 2024-09-13T13:07:38.724 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:38 smithi042 ceph-mon[63970]: Health check failed: Degraded data redundancy: 106/1599 objects degraded (6.629%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:38.724 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:38 smithi042 ceph-mon[63970]: from='client.34448 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:07:38.724 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:38 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1728395737' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:07:38.903 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 podman[65786]: 2024-09-13 13:07:38.636040006 +0000 UTC m=+0.137319543 container create 2852be73b94c7b0f8dbc8fdb6036273145b5ea52dc3ce75af8f7ccffd956db90 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-13T13:07:39.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 podman[65786]: 2024-09-13 13:07:38.902756469 +0000 UTC m=+0.404036036 container init 2852be73b94c7b0f8dbc8fdb6036273145b5ea52dc3ce75af8f7ccffd956db90 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-13T13:07:39.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:38 smithi046 podman[65786]: 2024-09-13 13:07:38.907255757 +0000 UTC m=+0.408535316 container start 2852be73b94c7b0f8dbc8fdb6036273145b5ea52dc3ce75af8f7ccffd956db90 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:07:39.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:39 smithi046 podman[65786]: 2024-09-13 13:07:39.002650414 +0000 UTC m=+0.503929964 container attach 2852be73b94c7b0f8dbc8fdb6036273145b5ea52dc3ce75af8f7ccffd956db90 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:07:39.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:39 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:39.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:39 smithi046 bash[65786]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:39.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:39 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:39.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:39 smithi046 bash[65786]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-13T13:07:39.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 248293 8.77 MB/sec execute 584 sec latency 38.610 ms 2024-09-13T13:07:40.516 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:40 smithi042 ceph-mon[63970]: pgmap v296: 97 pgs: 7 active+undersized, 4 peering, 31 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s wr, 288 op/s; 223/1524 objects degraded (14.633%) 2024-09-13T13:07:40.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 248562 8.77 MB/sec execute 585 sec latency 51.848 ms 2024-09-13T13:07:40.776 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:40 smithi046 ceph-mon[55972]: pgmap v296: 97 pgs: 7 active+undersized, 4 peering, 31 active+undersized+degraded, 55 active+clean; 353 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s wr, 288 op/s; 223/1524 objects degraded (14.633%) 2024-09-13T13:07:41.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 248857 8.77 MB/sec execute 586 sec latency 65.504 ms 2024-09-13T13:07:41.772 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T13:07:41.772 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T13:07:41.772 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T13:07:41.772 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T13:07:41.772 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T13:07:41.772 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-13T13:07:42.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate[65801]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T13:07:42.118 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 bash[65786]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-13T13:07:42.118 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:41 smithi046 podman[65786]: 2024-09-13 13:07:41.807913214 +0000 UTC m=+3.309192773 container died 2852be73b94c7b0f8dbc8fdb6036273145b5ea52dc3ce75af8f7ccffd956db90 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-13T13:07:42.628 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:42 smithi042 ceph-mon[63970]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 357 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.0 MiB/s wr, 391 op/s; 235/1527 objects degraded (15.390%) 2024-09-13T13:07:42.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 249120 8.77 MB/sec execute 587 sec latency 51.883 ms 2024-09-13T13:07:42.781 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:42 smithi046 ceph-mon[55972]: pgmap v297: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 357 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 6.5 KiB/s rd, 3.0 MiB/s wr, 391 op/s; 235/1527 objects degraded (15.390%) 2024-09-13T13:07:42.781 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:42 smithi046 podman[65960]: 2024-09-13 13:07:42.423676333 +0000 UTC m=+0.607941293 container remove 2852be73b94c7b0f8dbc8fdb6036273145b5ea52dc3ce75af8f7ccffd956db90 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-activate, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-13T13:07:42.781 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:42 smithi046 podman[66018]: 2024-09-13 13:07:42.54274455 +0000 UTC m=+0.017532116 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:07:42.781 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:42 smithi046 podman[66018]: 2024-09-13 13:07:42.657924508 +0000 UTC m=+0.132712065 container create 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:07:43.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:42 smithi046 podman[66018]: 2024-09-13 13:07:42.902676807 +0000 UTC m=+0.377464364 container init 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, org.label-schema.vendor=CentOS, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-13T13:07:43.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:42 smithi046 podman[66018]: 2024-09-13 13:07:42.906167011 +0000 UTC m=+0.380954568 container start 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731) 2024-09-13T13:07:43.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:42 smithi046 ceph-osd[66037]: -- 172.21.15.46:0/2388928054 <== mon.1 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b8a3aca1e0 con 0x55b8a3a56000 2024-09-13T13:07:43.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:43 smithi046 bash[66018]: 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 2024-09-13T13:07:43.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:43 smithi046 systemd[1]: Started Ceph osd.7 for e7c8d5a8-71cd-11ef-bceb-c7b262605968. 2024-09-13T13:07:43.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 249377 8.77 MB/sec execute 588 sec latency 69.363 ms 2024-09-13T13:07:44.152 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:44.152 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:44.152 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:44.153 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:44 smithi042 ceph-mon[63970]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 358 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 412 op/s; 258/1668 objects degraded (15.468%) 2024-09-13T13:07:44.153 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:44.153 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:44.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:44.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:44.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:44.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-mon[55972]: pgmap v298: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 358 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.8 KiB/s rd, 4.1 MiB/s wr, 412 op/s; 258/1668 objects degraded (15.468%) 2024-09-13T13:07:44.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:44.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:44.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 249636 8.77 MB/sec execute 589 sec latency 79.583 ms 2024-09-13T13:07:45.066 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:44 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:07:44.795+0000 7f5d5410e640 -1 Falling back to public interface 2024-09-13T13:07:45.172 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:45 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 258/1668 objects degraded (15.468%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:45.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:45 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 258/1668 objects degraded (15.468%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:45.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 249893 8.77 MB/sec execute 590 sec latency 66.493 ms 2024-09-13T13:07:46.198 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:46 smithi042 ceph-mon[63970]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 360 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.9 MiB/s wr, 419 op/s; 258/1662 objects degraded (15.523%) 2024-09-13T13:07:46.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:46 smithi046 ceph-mon[55972]: pgmap v299: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 360 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 4.8 KiB/s rd, 3.9 MiB/s wr, 419 op/s; 258/1662 objects degraded (15.523%) 2024-09-13T13:07:46.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 250204 8.76 MB/sec execute 591 sec latency 75.625 ms 2024-09-13T13:07:47.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 250470 8.75 MB/sec execute 592 sec latency 54.378 ms 2024-09-13T13:07:47.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.714 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.903 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.973 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.973 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:47.973 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:48.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 250762 8.75 MB/sec execute 593 sec latency 46.389 ms 2024-09-13T13:07:48.727 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:48 smithi042 ceph-mon[63970]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 359 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 258/1668 objects degraded (15.468%) 2024-09-13T13:07:48.993 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:48 smithi046 ceph-mon[55972]: pgmap v300: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 359 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 4.0 KiB/s rd, 4.2 MiB/s wr, 464 op/s; 258/1668 objects degraded (15.468%) 2024-09-13T13:07:49.687 INFO:teuthology.orchestra.run.smithi042.stdout: 5 251038 8.75 MB/sec execute 594 sec latency 57.379 ms 2024-09-13T13:07:49.763 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:49 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 245/1602 objects degraded (15.293%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:49.990 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:49 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 245/1602 objects degraded (15.293%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:50.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 251316 8.75 MB/sec execute 595 sec latency 76.621 ms 2024-09-13T13:07:50.795 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:50 smithi042 ceph-mon[63970]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 356 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 374 op/s; 245/1602 objects degraded (15.293%) 2024-09-13T13:07:51.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:50 smithi046 ceph-mon[55972]: pgmap v301: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 356 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 5.2 KiB/s rd, 4.0 MiB/s wr, 374 op/s; 245/1602 objects degraded (15.293%) 2024-09-13T13:07:51.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 251599 8.74 MB/sec execute 596 sec latency 45.914 ms 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:51.814 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T13:07:51.815 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T13:07:51.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T13:07:51.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T13:07:51.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T13:07:51.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T13:07:51.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T13:07:51.816 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:51 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:52.051 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-13T13:07:52.052 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-13T13:07:52.053 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:51 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-13T13:07:52.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:07:52.048+0000 7f5d5410e640 -1 osd.7 0 read_superblock omap replica is missing. 2024-09-13T13:07:52.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 251880 8.75 MB/sec execute 597 sec latency 94.570 ms 2024-09-13T13:07:52.822 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all osd 2024-09-13T13:07:52.822 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.1 MiB/s wr, 464 op/s; 246/1599 objects degraded (15.385%) 2024-09-13T13:07:52.822 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T13:07:52.822 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T13:07:52.822 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T13:07:52.822 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: osdmap e159: 8 total, 7 up, 8 in 2024-09-13T13:07:52.823 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi042.hljzvq"]} : dispatch 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all osd 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: pgmap v302: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 8.1 KiB/s rd, 4.1 MiB/s wr, 464 op/s; 246/1599 objects degraded (15.385%) 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: Upgrade: Setting require_osd_release to 19 squid 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: osdmap e159: 8 total, 7 up, 8 in 2024-09-13T13:07:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi042.hljzvq"]} : dispatch 2024-09-13T13:07:53.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 252083 8.74 MB/sec execute 598 sec latency 53.427 ms 2024-09-13T13:07:53.847 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:53 smithi042 ceph-mon[63970]: Upgrade: It appears safe to stop mds.foofs.smithi042.hljzvq 2024-09-13T13:07:54.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:53 smithi046 ceph-mon[55972]: Upgrade: It appears safe to stop mds.foofs.smithi042.hljzvq 2024-09-13T13:07:54.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:53 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:07:53.862+0000 7f5d5410e640 -1 osd.7 156 log_to_monitors true 2024-09-13T13:07:54.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 252318 8.74 MB/sec execute 599 sec latency 101.130 ms 2024-09-13T13:07:54.875 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:54 smithi042 ceph-mon[63970]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 421 op/s; 243/1575 objects degraded (15.429%) 2024-09-13T13:07:54.875 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:54 smithi042 ceph-mon[63970]: from='osd.7 [v2:172.21.15.46:6824/2931206155,v1:172.21.15.46:6825/2931206155]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T13:07:54.875 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:54 smithi042 ceph-mon[63970]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T13:07:55.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:07:54 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:07:54.820+0000 7f5d4beb8640 -1 osd.7 156 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T13:07:55.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:54 smithi046 ceph-mon[55972]: pgmap v304: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 7.2 KiB/s rd, 4.1 MiB/s wr, 421 op/s; 243/1575 objects degraded (15.429%) 2024-09-13T13:07:55.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:54 smithi046 ceph-mon[55972]: from='osd.7 [v2:172.21.15.46:6824/2931206155,v1:172.21.15.46:6825/2931206155]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T13:07:55.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:54 smithi046 ceph-mon[55972]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-13T13:07:55.688 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 600 sec 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: from='osd.7 [v2:172.21.15.46:6824/2931206155,v1:172.21.15.46:6825/2931206155]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: osdmap e160: 8 total, 7 up, 8 in 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: Upgrade: Updating mds.foofs.smithi042.hljzvq 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:55.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:07:55.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:55.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: Deploying daemon mds.foofs.smithi042.hljzvq on smithi042 2024-09-13T13:07:55.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: pgmap v306: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 425 op/s; 243/1581 objects degraded (15.370%) 2024-09-13T13:07:55.894 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:55 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 243/1575 objects degraded (15.429%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:55.994 INFO:teuthology.orchestra.run.smithi042.stdout: 0 cleanup 600 sec 2024-09-13T13:07:55.994 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T13:07:55.994 INFO:teuthology.orchestra.run.smithi042.stdout: Operation Count AvgLat MaxLat 2024-09-13T13:07:55.994 INFO:teuthology.orchestra.run.smithi042.stdout: ---------------------------------------- 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: NTCreateX 167884 4.984 2175.337 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Close 123321 0.141 29.331 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Rename 7118 10.289 1086.521 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Unlink 33899 7.492 2143.342 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Qpathinfo 152360 1.298 2138.008 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Qfileinfo 26587 0.007 1.232 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Qfsinfo 27908 1.474 4771.357 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Sfileinfo 13664 13.718 951.759 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: Find 58858 1.028 30.253 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: WriteX 82969 15.977 2720.292 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: ReadX 263751 0.010 5.225 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: LockX 550 0.383 2.450 2024-09-13T13:07:55.995 INFO:teuthology.orchestra.run.smithi042.stdout: UnlockX 550 0.344 3.267 2024-09-13T13:07:55.996 INFO:teuthology.orchestra.run.smithi042.stdout: Flush 11738 0.003 0.059 2024-09-13T13:07:55.996 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T13:07:55.996 INFO:teuthology.orchestra.run.smithi042.stdout:Throughput 8.73728 MB/sec 5 clients 5 procs max_latency=4771.366 ms 2024-09-13T13:07:55.997 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'umount /mnt/foo' 2024-09-13T13:07:56.051 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done' 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: from='osd.7 [v2:172.21.15.46:6824/2931206155,v1:172.21.15.46:6825/2931206155]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: osdmap e160: 8 total, 7 up, 8 in 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]} : dispatch 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: Upgrade: Updating mds.foofs.smithi042.hljzvq 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi042.hljzvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: Deploying daemon mds.foofs.smithi042.hljzvq on smithi042 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: pgmap v306: 97 pgs: 7 active+undersized, 35 active+undersized+degraded, 55 active+clean; 355 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 7.5 KiB/s rd, 4.1 MiB/s wr, 425 op/s; 243/1581 objects degraded (15.370%) 2024-09-13T13:07:56.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:55 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 243/1575 objects degraded (15.429%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T13:07:56.202 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'cd /mnt/foo && dbench 5 -t 5' 2024-09-13T13:07:56.245 INFO:teuthology.orchestra.run.smithi042.stdout:dbench version 4.00 - Copyright Andrew Tridgell 1999-2004 2024-09-13T13:07:56.245 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T13:07:56.246 INFO:teuthology.orchestra.run.smithi042.stdout:Running for 5 seconds with load '/usr/share/dbench/client.txt' and minimum warmup 1 secs 2024-09-13T13:07:56.246 INFO:teuthology.orchestra.run.smithi042.stdout:3 of 5 processes prepared for launch 0 sec 2024-09-13T13:07:56.346 INFO:teuthology.orchestra.run.smithi042.stdout:5 of 5 processes prepared for launch 0 sec 2024-09-13T13:07:56.346 INFO:teuthology.orchestra.run.smithi042.stdout:releasing clients 2024-09-13T13:07:57.083 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:56 smithi042 ceph-mon[63970]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:07:57.083 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:56 smithi042 ceph-mon[63970]: osd.7 [v2:172.21.15.46:6824/2931206155,v1:172.21.15.46:6825/2931206155] boot 2024-09-13T13:07:57.083 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:56 smithi042 ceph-mon[63970]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T13:07:57.083 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T13:07:57.084 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:56 smithi042 ceph-mon[63970]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T13:07:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:56 smithi046 ceph-mon[55972]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T13:07:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:56 smithi046 ceph-mon[55972]: osd.7 [v2:172.21.15.46:6824/2931206155,v1:172.21.15.46:6825/2931206155] boot 2024-09-13T13:07:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:56 smithi046 ceph-mon[55972]: osdmap e161: 8 total, 8 up, 8 in 2024-09-13T13:07:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-13T13:07:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:56 smithi046 ceph-mon[55972]: osdmap e162: 8 total, 8 up, 8 in 2024-09-13T13:07:57.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[63966]: 2024-09-13T13:07:57.618+0000 7fa930a6d640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: pgmap v309: 97 pgs: 15 peering, 2 active+undersized, 25 active+undersized+degraded, 55 active+clean; 362 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 435 op/s; 178/1626 objects degraded (10.947%) 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: Standby daemon mds.foofs.smithi046.nqnolk assigned to filesystem foofs as rank 0 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T13:07:57.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:57 smithi042 ceph-mon[63970]: fsmap foofs:1/1 {0=foofs.smithi046.nqnolk=up:replay} 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: pgmap v309: 97 pgs: 15 peering, 2 active+undersized, 25 active+undersized+degraded, 55 active+clean; 362 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.3 KiB/s rd, 4.4 MiB/s wr, 435 op/s; 178/1626 objects degraded (10.947%) 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: osdmap e163: 8 total, 8 up, 8 in 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: Standby daemon mds.foofs.smithi046.nqnolk assigned to filesystem foofs as rank 0 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T13:07:58.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T13:07:58.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:57 smithi046 ceph-mon[55972]: fsmap foofs:1/1 {0=foofs.smithi046.nqnolk=up:replay} 2024-09-13T13:07:58.346 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83 4.45 MB/sec execute 1 sec latency 1102.082 ms 2024-09-13T13:07:59.347 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83 2.23 MB/sec execute 2 sec latency 1663.791 ms 2024-09-13T13:07:59.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:59 smithi042 ceph-mon[63970]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T13:07:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:07:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:07:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:07:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:59 smithi046 ceph-mon[55972]: osdmap e164: 8 total, 8 up, 8 in 2024-09-13T13:08:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:07:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:00.346 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83 1.48 MB/sec execute 3 sec latency 2663.835 ms 2024-09-13T13:08:00.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:00 smithi042 ceph-mon[63970]: pgmap v312: 97 pgs: 1 active+undersized+remapped, 15 peering, 2 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 63 active+clean; 366 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.8 MiB/s wr, 201 op/s; 113/1629 objects degraded (6.937%); 397 KiB/s, 18 objects/s recovering 2024-09-13T13:08:00.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:00 smithi042 ceph-mon[63970]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T13:08:00.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:00 smithi042 ceph-mon[63970]: Health check update: Degraded data redundancy: 113/1629 objects degraded (6.937%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:08:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:00 smithi046 ceph-mon[55972]: pgmap v312: 97 pgs: 1 active+undersized+remapped, 15 peering, 2 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 63 active+clean; 366 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.8 MiB/s wr, 201 op/s; 113/1629 objects degraded (6.937%); 397 KiB/s, 18 objects/s recovering 2024-09-13T13:08:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:00 smithi046 ceph-mon[55972]: osdmap e165: 8 total, 8 up, 8 in 2024-09-13T13:08:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:00 smithi046 ceph-mon[55972]: Health check update: Degraded data redundancy: 113/1629 objects degraded (6.937%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T13:08:01.346 INFO:teuthology.orchestra.run.smithi042.stdout: 5 83 1.11 MB/sec execute 4 sec latency 3663.876 ms 2024-09-13T13:08:01.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:01 smithi042 ceph-mon[63970]: reconnect by client.14658 172.21.15.42:0/1424543809 after 0.00099999 2024-09-13T13:08:01.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:01 smithi042 ceph-mon[63970]: reconnect by client.24347 172.21.15.46:0/2863232647 after 0.00099999 2024-09-13T13:08:01.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:01 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:reconnect 2024-09-13T13:08:01.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:01 smithi042 ceph-mon[63970]: fsmap foofs:1/1 {0=foofs.smithi046.nqnolk=up:reconnect} 2024-09-13T13:08:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:01 smithi046 ceph-mon[55972]: reconnect by client.14658 172.21.15.42:0/1424543809 after 0.00099999 2024-09-13T13:08:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:01 smithi046 ceph-mon[55972]: reconnect by client.24347 172.21.15.46:0/2863232647 after 0.00099999 2024-09-13T13:08:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:01 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:reconnect 2024-09-13T13:08:02.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:01 smithi046 ceph-mon[55972]: fsmap foofs:1/1 {0=foofs.smithi046.nqnolk=up:reconnect} 2024-09-13T13:08:02.346 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 5 sec 2024-09-13T13:08:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: pgmap v314: 97 pgs: 12 peering, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 370 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 3.3 MiB/s wr, 56 op/s; 8/1659 objects degraded (0.482%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T13:08:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T13:08:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:rejoin 2024-09-13T13:08:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.42:6834/379584727,v1:172.21.15.42:6835/379584727] up:boot 2024-09-13T13:08:02.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: fsmap foofs:1/1 {0=foofs.smithi046.nqnolk=up:rejoin} 1 up:standby 2024-09-13T13:08:02.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:02.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:02 smithi042 ceph-mon[63970]: daemon mds.foofs.smithi046.nqnolk is now active in filesystem foofs as rank 0 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: pgmap v314: 97 pgs: 12 peering, 2 active+recovering+undersized+degraded+remapped, 83 active+clean; 370 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 3.3 MiB/s wr, 56 op/s; 8/1659 objects degraded (0.482%); 2.3 MiB/s, 36 objects/s recovering 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:rejoin 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.42:6834/379584727,v1:172.21.15.42:6835/379584727] up:boot 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: fsmap foofs:1/1 {0=foofs.smithi046.nqnolk=up:rejoin} 1 up:standby 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:02 smithi046 ceph-mon[55972]: daemon mds.foofs.smithi046.nqnolk is now active in filesystem foofs as rank 0 2024-09-13T13:08:03.348 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 6 sec 2024-09-13T13:08:03.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:03 smithi042 ceph-mon[63970]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T13:08:03.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:03 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:active 2024-09-13T13:08:03.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:03 smithi042 ceph-mon[63970]: fsmap foofs:1 {0=foofs.smithi046.nqnolk=up:active} 1 up:standby 2024-09-13T13:08:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:03 smithi046 ceph-mon[55972]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T13:08:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:03 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.46:6832/2390786867,v1:172.21.15.46:6833/2390786867] up:active 2024-09-13T13:08:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:03 smithi046 ceph-mon[55972]: fsmap foofs:1 {0=foofs.smithi046.nqnolk=up:active} 1 up:standby 2024-09-13T13:08:04.347 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 7 sec 2024-09-13T13:08:04.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:04 smithi042 ceph-mon[63970]: pgmap v315: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 3.6 MiB/s wr, 69 op/s; 2.1 MiB/s, 45 objects/s recovering 2024-09-13T13:08:04.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:04 smithi042 ceph-mon[63970]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/1659 objects degraded (0.482%), 2 pgs degraded) 2024-09-13T13:08:04.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:04 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:08:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:04 smithi046 ceph-mon[55972]: pgmap v315: 97 pgs: 97 active+clean; 352 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 3.6 MiB/s wr, 69 op/s; 2.1 MiB/s, 45 objects/s recovering 2024-09-13T13:08:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:04 smithi046 ceph-mon[55972]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/1659 objects degraded (0.482%), 2 pgs degraded) 2024-09-13T13:08:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:04 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:08:05.347 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 8 sec 2024-09-13T13:08:06.347 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 9 sec 2024-09-13T13:08:06.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:06 smithi042 ceph-mon[63970]: pgmap v316: 97 pgs: 97 active+clean; 320 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 68 op/s; 1.7 MiB/s, 35 objects/s recovering 2024-09-13T13:08:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:06 smithi046 ceph-mon[55972]: pgmap v316: 97 pgs: 97 active+clean; 320 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 68 op/s; 1.7 MiB/s, 35 objects/s recovering 2024-09-13T13:08:07.347 INFO:teuthology.orchestra.run.smithi042.stdout: 5 cleanup 10 sec 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: 0 cleanup 10 sec 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: Operation Count AvgLat MaxLat 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: ---------------------------------------- 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: Close 5 9804.380 9808.879 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: Qfileinfo 5 806.207 1102.075 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: WriteX 68 0.034 0.077 2024-09-13T13:08:07.574 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T13:08:07.575 INFO:teuthology.orchestra.run.smithi042.stdout:Throughput 1.11355 MB/sec 5 clients 5 procs max_latency=3663.876 ms 2024-09-13T13:08:07.576 DEBUG:teuthology.parallel:result is None 2024-09-13T13:08:08.918 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:08 smithi042 ceph-mon[63970]: pgmap v317: 97 pgs: 97 active+clean; 276 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.7 MiB/s wr, 71 op/s; 1.3 MiB/s, 22 objects/s recovering 2024-09-13T13:08:08.918 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:08:08.919 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:08 smithi046 ceph-mon[55972]: pgmap v317: 97 pgs: 97 active+clean; 276 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 MiB/s rd, 1.7 MiB/s wr, 71 op/s; 1.3 MiB/s, 22 objects/s recovering 2024-09-13T13:08:09.717 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:08:09.717 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (9m) 2m ago 21m 19.8M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (6m) 2m ago 21m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (6m) 23s ago 18m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (9m) 2m ago 18m 45.1M - dad864ee21e9 7144014c89b8 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (12m) 2m ago 12m 40.9M - 17.2.0 e1d6a67b021e d21b8f70cfc6 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (12m) 23s ago 12m 16.7M - 17.2.0 e1d6a67b021e 2fef7efee654 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (8m) 2m ago 23m 519M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (7m) 23s ago 18m 456M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (7m) 2m ago 23m 56.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (7m) 23s ago 18m 50.1M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (12m) 2m ago 12m 267M - 4.0 e1d6a67b021e 4568d79739de 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (12m) 23s ago 12m 91.0M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (9m) 2m ago 18m 10.7M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (8m) 23s ago 18m 11.2M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:08:09.718 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (5m) 2m ago 17m 335M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (4m) 2m ago 16m 254M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (3m) 2m ago 16m 179M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (3m) 2m ago 15m 11.9M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (2m) 23s ago 15m 346M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (103s) 23s ago 15m 283M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (65s) 23s ago 14m 249M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 9d85dde44d01 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (27s) 23s ago 14m 11.7M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 465a73580a0c 2024-09-13T13:08:09.719 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (8m) 2m ago 17m 55.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:08:10.187 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:08:10.187 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:08:10.187 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:08:10.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:10.188 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:08:10.188 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:08:10.188 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:10.189 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:08:10.189 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:08:10.189 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:10.189 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:08:10.189 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T13:08:10.189 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 1 2024-09-13T13:08:10.190 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:10.190 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:08:10.190 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-09-13T13:08:10.190 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 13 2024-09-13T13:08:10.190 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:08:10.191 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: pgmap v318: 97 pgs: 97 active+clean; 266 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.5 MiB/s wr, 65 op/s; 1.1 MiB/s, 19 objects/s recovering 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: daemon mds.foofs.smithi042.hljzvq restarted 2024-09-13T13:08:10.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:10.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T13:08:10.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:10.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:10 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:10.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='client.34468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: pgmap v318: 97 pgs: 97 active+clean; 266 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.5 MiB/s wr, 65 op/s; 1.1 MiB/s, 19 objects/s recovering 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='client.34472 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: daemon mds.foofs.smithi042.hljzvq restarted 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: osdmap e166: 8 total, 8 up, 8 in 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:10.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:10 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:10.569 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:08:10.569 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:08:10.569 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:08:10.569 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:08:10.569 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:08:10.569 INFO:teuthology.orchestra.run.smithi042.stdout: "osd", 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "14/23 daemons upgraded", 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading mds daemons", 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:08:10.570 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:08:11.021 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:08:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:11 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:boot 2024-09-13T13:08:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:11 smithi046 ceph-mon[55972]: fsmap foofs:1 {0=foofs.smithi046.nqnolk=up:active} 1 up:standby 2024-09-13T13:08:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:11 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:11 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/3501507978' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:11 smithi046 ceph-mon[55972]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:11.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:11 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2074360370' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:08:11.409 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:11 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:boot 2024-09-13T13:08:11.409 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:11 smithi042 ceph-mon[63970]: fsmap foofs:1 {0=foofs.smithi046.nqnolk=up:active} 1 up:standby 2024-09-13T13:08:11.409 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:11 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:11.409 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:11 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/3501507978' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:11.410 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:11 smithi042 ceph-mon[63970]: from='client.34484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:11.410 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:11 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2074360370' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:08:12.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:12 smithi046 ceph-mon[55972]: pgmap v320: 97 pgs: 97 active+clean; 262 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 683 KiB/s wr, 55 op/s; 253 KiB/s, 10 objects/s recovering 2024-09-13T13:08:12.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:12 smithi042 ceph-mon[63970]: pgmap v320: 97 pgs: 97 active+clean; 262 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 683 KiB/s wr, 55 op/s; 253 KiB/s, 10 objects/s recovering 2024-09-13T13:08:14.604 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:14 smithi042 ceph-mon[63970]: pgmap v321: 97 pgs: 97 active+clean; 252 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 60 KiB/s wr, 47 op/s 2024-09-13T13:08:14.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:14.605 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:14 smithi046 ceph-mon[55972]: pgmap v321: 97 pgs: 97 active+clean; 252 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 KiB/s rd, 60 KiB/s wr, 47 op/s 2024-09-13T13:08:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:15 smithi046 ceph-mon[55972]: pgmap v322: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 57 KiB/s wr, 36 op/s 2024-09-13T13:08:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:15 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.150 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.151 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.151 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:15 smithi042 ceph-mon[63970]: pgmap v322: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 7.3 KiB/s rd, 57 KiB/s wr, 36 op/s 2024-09-13T13:08:16.151 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:16.151 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:15 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:18.655 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:18 smithi042 ceph-mon[63970]: pgmap v323: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 59 KiB/s wr, 16 op/s 2024-09-13T13:08:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:18 smithi046 ceph-mon[55972]: pgmap v323: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 59 KiB/s wr, 16 op/s 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: Detected new or changed devices on smithi042 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: pgmap v324: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 59 KiB/s wr, 13 op/s 2024-09-13T13:08:20.193 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:20.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi046.nqnolk"]} : dispatch 2024-09-13T13:08:20.194 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:20 smithi046 ceph-mon[55972]: Upgrade: It appears safe to stop mds.foofs.smithi046.nqnolk 2024-09-13T13:08:20.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: Detected new or changed devices on smithi042 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: pgmap v324: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.4 KiB/s rd, 59 KiB/s wr, 13 op/s 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:20.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds ok-to-stop", "ids": ["foofs.smithi046.nqnolk"]} : dispatch 2024-09-13T13:08:20.493 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:20 smithi042 ceph-mon[63970]: Upgrade: It appears safe to stop mds.foofs.smithi046.nqnolk 2024-09-13T13:08:22.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:21 smithi042 ceph-mon[63970]: Upgrade: Updating mds.foofs.smithi046.nqnolk 2024-09-13T13:08:22.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:22.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:08:22.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:21 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:22.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:21 smithi042 ceph-mon[63970]: Deploying daemon mds.foofs.smithi046.nqnolk on smithi046 2024-09-13T13:08:22.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:21 smithi042 ceph-mon[63970]: pgmap v325: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 49 KiB/s wr, 9 op/s 2024-09-13T13:08:22.244 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:21 smithi046 ceph-mon[55972]: Upgrade: Updating mds.foofs.smithi046.nqnolk 2024-09-13T13:08:22.245 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:22.245 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "mds.foofs.smithi046.nqnolk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-13T13:08:22.245 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:21 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:22.245 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:21 smithi046 ceph-mon[55972]: Deploying daemon mds.foofs.smithi046.nqnolk on smithi046 2024-09-13T13:08:22.245 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:21 smithi046 ceph-mon[55972]: pgmap v325: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.9 KiB/s rd, 49 KiB/s wr, 9 op/s 2024-09-13T13:08:23.117 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:08:22.823+0000 7f5d36673640 -1 bdev(0x55b8a3abec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-09-13T13:08:23.118 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:08:22.824+0000 7f5d38e78640 -1 bdev(0x55b8a3abec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 1 2024-09-13T13:08:23.118 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:08:22.824+0000 7f5d37e76640 -1 bdev(0x55b8a3abec00 /var/lib/ceph/osd/ceph-7/block) aio_submit retries 2 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: Standby daemon mds.foofs.smithi042.hljzvq assigned to filesystem foofs as rank 0 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T13:08:23.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T13:08:23.119 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:22 smithi046 ceph-mon[55972]: fsmap foofs:1/1 {0=foofs.smithi042.hljzvq=up:replay} 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[63966]: 2024-09-13T13:08:22.807+0000 7fa930a6d640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: osdmap e167: 8 total, 8 up, 8 in 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: Standby daemon mds.foofs.smithi042.hljzvq assigned to filesystem foofs as rank 0 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: fsmap foofs:0/1 1 up:standby, 1 failed 2024-09-13T13:08:23.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:22 smithi042 ceph-mon[63970]: fsmap foofs:1/1 {0=foofs.smithi042.hljzvq=up:replay} 2024-09-13T13:08:24.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:23 smithi042 ceph-mon[63970]: pgmap v327: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.2 KiB/s wr, 2 op/s 2024-09-13T13:08:24.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:23 smithi046 ceph-mon[55972]: pgmap v327: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.2 KiB/s wr, 2 op/s 2024-09-13T13:08:25.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:24 smithi042 ceph-mon[63970]: reconnect by client.14658 172.21.15.42:0/1424543809 after 0.000999992 2024-09-13T13:08:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:24 smithi042 ceph-mon[63970]: reconnect by client.24347 172.21.15.46:0/2863232647 after 0.000999992 2024-09-13T13:08:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:24 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:reconnect 2024-09-13T13:08:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:24 smithi042 ceph-mon[63970]: fsmap foofs:1/1 {0=foofs.smithi042.hljzvq=up:reconnect} 2024-09-13T13:08:25.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:08:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:24 smithi046 ceph-mon[55972]: reconnect by client.14658 172.21.15.42:0/1424543809 after 0.000999992 2024-09-13T13:08:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:24 smithi046 ceph-mon[55972]: reconnect by client.24347 172.21.15.46:0/2863232647 after 0.000999992 2024-09-13T13:08:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:24 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:reconnect 2024-09-13T13:08:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:24 smithi046 ceph-mon[55972]: fsmap foofs:1/1 {0=foofs.smithi042.hljzvq=up:reconnect} 2024-09-13T13:08:25.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:08:26.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:25 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:rejoin 2024-09-13T13:08:26.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:25 smithi042 ceph-mon[63970]: fsmap foofs:1/1 {0=foofs.smithi042.hljzvq=up:rejoin} 2024-09-13T13:08:26.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:25 smithi042 ceph-mon[63970]: daemon mds.foofs.smithi042.hljzvq is now active in filesystem foofs as rank 0 2024-09-13T13:08:26.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:25 smithi042 ceph-mon[63970]: pgmap v328: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 KiB/s wr, 4 op/s 2024-09-13T13:08:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:25 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:rejoin 2024-09-13T13:08:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:25 smithi046 ceph-mon[55972]: fsmap foofs:1/1 {0=foofs.smithi042.hljzvq=up:rejoin} 2024-09-13T13:08:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:25 smithi046 ceph-mon[55972]: daemon mds.foofs.smithi042.hljzvq is now active in filesystem foofs as rank 0 2024-09-13T13:08:26.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:25 smithi046 ceph-mon[55972]: pgmap v328: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 3.9 KiB/s wr, 4 op/s 2024-09-13T13:08:27.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:26 smithi042 ceph-mon[63970]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T13:08:27.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:26 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:active 2024-09-13T13:08:27.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:26 smithi042 ceph-mon[63970]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 2024-09-13T13:08:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:26 smithi046 ceph-mon[55972]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T13:08:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:26 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012] up:active 2024-09-13T13:08:27.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:26 smithi046 ceph-mon[55972]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 2024-09-13T13:08:28.109 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:27 smithi046 ceph-mon[55972]: pgmap v329: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.4 KiB/s wr, 8 op/s 2024-09-13T13:08:28.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:27 smithi042 ceph-mon[63970]: pgmap v329: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 1.4 KiB/s wr, 8 op/s 2024-09-13T13:08:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:29 smithi046 ceph-mon[55972]: pgmap v330: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-09-13T13:08:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:30.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:30.135 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:30.135 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:30.135 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:29 smithi042 ceph-mon[63970]: pgmap v330: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-09-13T13:08:30.135 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:30.136 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:30.136 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:31.096 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:30 smithi046 ceph-mon[55972]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T13:08:31.096 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:30 smithi046 ceph-mon[55972]: Cluster is now healthy 2024-09-13T13:08:31.097 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:30 smithi046 ceph-mon[55972]: mds.? [v2:172.21.15.46:6832/2821764532,v1:172.21.15.46:6833/2821764532] up:boot 2024-09-13T13:08:31.097 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:30 smithi046 ceph-mon[55972]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T13:08:31.097 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:30 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:08:31.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:30 smithi042 ceph-mon[63970]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T13:08:31.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:30 smithi042 ceph-mon[63970]: Cluster is now healthy 2024-09-13T13:08:31.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:30 smithi042 ceph-mon[63970]: mds.? [v2:172.21.15.46:6832/2821764532,v1:172.21.15.46:6833/2821764532] up:boot 2024-09-13T13:08:31.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:30 smithi042 ceph-mon[63970]: fsmap foofs:1 {0=foofs.smithi042.hljzvq=up:active} 1 up:standby 2024-09-13T13:08:31.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:30 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "mds metadata", "who": "foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:08:32.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:31 smithi046 ceph-mon[55972]: pgmap v331: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 1.5 KiB/s wr, 12 op/s 2024-09-13T13:08:32.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:31 smithi042 ceph-mon[63970]: pgmap v331: 97 pgs: 97 active+clean; 248 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 1.5 KiB/s wr, 12 op/s 2024-09-13T13:08:33.726 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:33 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.727 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:33 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.727 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:33 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.727 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:33 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:33 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:33 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:33 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:33.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:33 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:34.538 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:34 smithi046 ceph-mon[55972]: pgmap v332: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.2 KiB/s wr, 14 op/s 2024-09-13T13:08:34.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:34 smithi042 ceph-mon[63970]: pgmap v332: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 8.2 KiB/s wr, 14 op/s 2024-09-13T13:08:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: pgmap v333: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T13:08:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:08:36.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi042.hljzvq"}]': finished 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi046.nqnolk"}]': finished 2024-09-13T13:08:36.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.744 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:36 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: pgmap v333: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 19 MiB/s rd, 7.2 KiB/s wr, 12 op/s 2024-09-13T13:08:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi042.hljzvq"} : dispatch 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi042.hljzvq"}]': finished 2024-09-13T13:08:36.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi046.nqnolk"} : dispatch 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.foofs.smithi046.nqnolk"}]': finished 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:36.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:36 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:37.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:37 smithi042 ceph-mon[63970]: Detected new or changed devices on smithi046 2024-09-13T13:08:37.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:37 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all mds 2024-09-13T13:08:37.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:37 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all rgw 2024-09-13T13:08:37.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:37 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T13:08:37.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:37 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T13:08:37.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:37 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all iscsi 2024-09-13T13:08:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:37 smithi046 ceph-mon[55972]: Detected new or changed devices on smithi046 2024-09-13T13:08:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:37 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all mds 2024-09-13T13:08:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:37 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all rgw 2024-09-13T13:08:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:37 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all rbd-mirror 2024-09-13T13:08:37.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:37 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all ceph-exporter 2024-09-13T13:08:37.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:37 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all iscsi 2024-09-13T13:08:38.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: pgmap v334: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-09-13T13:08:38.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:38.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T13:08:38.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T13:08:38.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T13:08:38.699 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:38.700 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T13:08:38.700 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T13:08:38.700 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T13:08:38.700 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:38 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:38.769 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: pgmap v334: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 7.2 KiB/s wr, 10 op/s 2024-09-13T13:08:38.770 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:38.770 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T13:08:38.770 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T13:08:38.770 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T13:08:38.770 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:38.770 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T13:08:38.771 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T13:08:38.771 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.0.0.smithi042.fnqkqg-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T13:08:38.771 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:38 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:39.706 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Upgrade: Updating nfs.foo.0.0.smithi042.fnqkqg (1/2) 2024-09-13T13:08:39.706 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg 2024-09-13T13:08:39.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T13:08:39.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Rados config object exists: conf-nfs.foo 2024-09-13T13:08:39.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg-rgw 2024-09-13T13:08:39.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Bind address in nfs.foo.0.0.smithi042.fnqkqg's ganesha conf is defaulting to empty 2024-09-13T13:08:39.707 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:39 smithi042 ceph-mon[63970]: Deploying daemon nfs.foo.0.0.smithi042.fnqkqg on smithi042 2024-09-13T13:08:39.780 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Upgrade: Updating nfs.foo.0.0.smithi042.fnqkqg (1/2) 2024-09-13T13:08:39.780 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg 2024-09-13T13:08:39.780 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Ensuring nfs.foo.0 is in the ganesha grace table 2024-09-13T13:08:39.781 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Rados config object exists: conf-nfs.foo 2024-09-13T13:08:39.781 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Creating key for client.nfs.foo.0.0.smithi042.fnqkqg-rgw 2024-09-13T13:08:39.781 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Bind address in nfs.foo.0.0.smithi042.fnqkqg's ganesha conf is defaulting to empty 2024-09-13T13:08:39.781 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:39 smithi046 ceph-mon[55972]: Deploying daemon nfs.foo.0.0.smithi042.fnqkqg on smithi042 2024-09-13T13:08:40.701 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:40 smithi042 ceph-mon[63970]: pgmap v335: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-09-13T13:08:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:40 smithi046 ceph-mon[55972]: pgmap v335: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 7.2 KiB/s wr, 6 op/s 2024-09-13T13:08:41.437 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:08:42.170 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:08:42.170 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (10m) 27s ago 22m 24.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:08:42.170 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (6m) 27s ago 22m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (6m) 9s ago 18m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (10m) 27s ago 19m 52.3M - dad864ee21e9 7144014c89b8 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (32s) 27s ago 13m 13.0M - 19.3.0-4917-gb0201b8c 1641fe80ce72 a4b67e82e926 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (13s) 9s ago 13m 15.4M - 19.3.0-4917-gb0201b8c 1641fe80ce72 f65a07dc3c16 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (9m) 27s ago 23m 523M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (8m) 9s ago 18m 457M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (8m) 27s ago 23m 66.0M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (7m) 9s ago 18m 51.3M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 starting - - - - 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (12m) 9s ago 12m 91.1M - 4.0 e1d6a67b021e fdea9e008196 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (9m) 27s ago 19m 10.1M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (9m) 9s ago 18m 10.9M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (6m) 27s ago 17m 379M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:08:42.171 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (5m) 27s ago 17m 403M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (4m) 27s ago 16m 331M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (3m) 27s ago 16m 371M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (2m) 9s ago 16m 354M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (2m) 9s ago 15m 299M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (97s) 9s ago 15m 264M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 9d85dde44d01 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (59s) 9s ago 14m 278M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 465a73580a0c 2024-09-13T13:08:42.172 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (8m) 27s ago 18m 57.7M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:08:42.622 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:42 smithi046 ceph-mon[55972]: pgmap v336: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-13T13:08:42.623 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:42 smithi046 ceph-mon[55972]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:42.623 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:42 smithi046 ceph-mon[55972]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:42.623 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:42 smithi046 ceph-mon[55972]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.42:0/1424543809"} : dispatch 2024-09-13T13:08:42.623 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:42.623 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:42 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:42.638 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:08:42.639 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:08:42.639 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:08:42.639 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:08:42.639 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 14 2024-09-13T13:08:42.639 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:08:42.639 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:08:42.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:42 smithi042 ceph-mon[63970]: pgmap v336: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 7.2 KiB/s wr, 5 op/s 2024-09-13T13:08:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:42 smithi042 ceph-mon[63970]: from='client.34504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:42 smithi042 ceph-mon[63970]: from='client.34508 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:42 smithi042 ceph-mon[63970]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.42:0/1424543809"} : dispatch 2024-09-13T13:08:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:42 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "mds", 2024-09-13T13:08:43.001 INFO:teuthology.orchestra.run.smithi042.stdout: "osd", 2024-09-13T13:08:43.002 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:08:43.002 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:08:43.002 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "16/23 daemons upgraded", 2024-09-13T13:08:43.002 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading nfs daemons", 2024-09-13T13:08:43.002 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:08:43.002 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:08:43.457 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: Evicting (and blocklisting) client session 14658 (172.21.15.42:0/1424543809) 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.42:0/1424543809"}]': finished 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/993774844' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:43.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T13:08:43.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T13:08:43.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T13:08:43.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:43.743 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:43 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/4176565638' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:08:43.812 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: Evicting (and blocklisting) client session 14658 (172.21.15.42:0/1424543809) 2024-09-13T13:08:43.812 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='client.34520 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:43.812 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.42:0/1424543809"}]': finished 2024-09-13T13:08:43.812 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: osdmap e168: 8 total, 8 up, 8 in 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/993774844' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb", "caps": ["mon", "allow r", "osd", "allow rw pool=.nfs namespace=foo"]} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth get-or-create", "entity": "client.mgr.nfs.grace.nfs.foo", "caps": ["mon", "allow r", "osd", "allow rwx pool .nfs"]}]': finished 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "auth rm", "entity": "client.mgr.nfs.grace.nfs.foo"}]': finished 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get-or-create", "entity": "client.nfs.foo.1.0.smithi046.jjnsrb-rgw", "caps": ["mon", "allow r", "osd", "allow rwx tag rgw *=*"]} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:43.813 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:43 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/4176565638' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:08:44.692 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Upgrade: Updating nfs.foo.1.0.smithi046.jjnsrb (2/2) 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Rados config object exists: conf-nfs.foo 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb-rgw 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Bind address in nfs.foo.1.0.smithi046.jjnsrb's ganesha conf is defaulting to empty 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: Deploying daemon nfs.foo.1.0.smithi046.jjnsrb on smithi046 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: pgmap v338: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 819 B/s wr, 3 op/s 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:44.693 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:44.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: from='client.34532 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Upgrade: Updating nfs.foo.1.0.smithi046.jjnsrb (2/2) 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Ensuring nfs.foo.1 is in the ganesha grace table 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Rados config object exists: conf-nfs.foo 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Creating key for client.nfs.foo.1.0.smithi046.jjnsrb-rgw 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Bind address in nfs.foo.1.0.smithi046.jjnsrb's ganesha conf is defaulting to empty 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: Deploying daemon nfs.foo.1.0.smithi046.jjnsrb on smithi046 2024-09-13T13:08:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: pgmap v338: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 819 B/s wr, 3 op/s 2024-09-13T13:08:44.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:44.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:46 smithi046 ceph-mon[55972]: pgmap v339: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 921 B/s wr, 3 op/s 2024-09-13T13:08:46.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:46 smithi042 ceph-mon[63970]: pgmap v339: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 921 B/s wr, 3 op/s 2024-09-13T13:08:48.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:48.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:48.080 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:47 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:48.236 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:48.236 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:48.236 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:47 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:49.099 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:48 smithi046 ceph-mon[55972]: pgmap v340: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T13:08:49.241 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:48 smithi042 ceph-mon[63970]: pgmap v340: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.6 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T13:08:50.718 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: pgmap v341: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-13T13:08:50.718 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.719 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.719 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: Evicting (and blocklisting) client session 24347 (172.21.15.46:0/2863232647) 2024-09-13T13:08:50.719 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.46:0/2863232647"} : dispatch 2024-09-13T13:08:50.719 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.719 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:50 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: pgmap v341: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.1 KiB/s rd, 2.2 KiB/s wr, 7 op/s 2024-09-13T13:08:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: Evicting (and blocklisting) client session 24347 (172.21.15.46:0/2863232647) 2024-09-13T13:08:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd={"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.46:0/2863232647"} : dispatch 2024-09-13T13:08:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:50.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:50 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:51.451 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:51 smithi042 ceph-mon[63970]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.46:0/2863232647"}]': finished 2024-09-13T13:08:51.451 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:51 smithi042 ceph-mon[63970]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T13:08:51.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:51 smithi046 ceph-mon[55972]: from='mds.? [v2:172.21.15.42:6834/4146423012,v1:172.21.15.42:6835/4146423012]' entity='mds.foofs.smithi042.hljzvq' cmd='[{"prefix":"osd blocklist", "blocklistop":"add","addr":"172.21.15.46:0/2863232647"}]': finished 2024-09-13T13:08:51.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:51 smithi046 ceph-mon[55972]: osdmap e169: 8 total, 8 up, 8 in 2024-09-13T13:08:52.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:52 smithi042 ceph-mon[63970]: pgmap v343: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.3 KiB/s wr, 9 op/s 2024-09-13T13:08:52.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:52.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:52.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:52.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:52 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:53.013 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:52 smithi046 ceph-mon[55972]: pgmap v343: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 5.9 KiB/s rd, 2.3 KiB/s wr, 9 op/s 2024-09-13T13:08:53.014 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:53.014 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:53.014 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:53.014 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:52 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:54.186 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:54 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:54.186 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:54 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:54.186 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:54 smithi042 ceph-mon[63970]: pgmap v344: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-09-13T13:08:54.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:54 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:54.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:54 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:54.368 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:54 smithi046 ceph-mon[55972]: pgmap v344: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.8 KiB/s wr, 5 op/s 2024-09-13T13:08:56.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: pgmap v345: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.7 KiB/s wr, 5 op/s 2024-09-13T13:08:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:08:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:56 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: pgmap v345: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.7 KiB/s wr, 5 op/s 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:56 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.556 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi042.fnqkqg"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi042.fnqkqg"}]': finished 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi046.jjnsrb"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi046.jjnsrb"}]': finished 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.557 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:57 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:08:57.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi042.fnqkqg"} : dispatch 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.0.0.smithi042.fnqkqg"}]': finished 2024-09-13T13:08:57.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi046.jjnsrb"} : dispatch 2024-09-13T13:08:57.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs.foo.1.0.smithi046.jjnsrb"}]': finished 2024-09-13T13:08:57.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:08:57.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:57.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:57 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:08:58.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:58 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all nfs 2024-09-13T13:08:58.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:58 smithi042 ceph-mon[63970]: Upgrade: Setting container_image for all nvmeof 2024-09-13T13:08:58.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:58 smithi042 ceph-mon[63970]: pgmap v346: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T13:08:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:58 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all nfs 2024-09-13T13:08:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:58 smithi046 ceph-mon[55972]: Upgrade: Setting container_image for all nvmeof 2024-09-13T13:08:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:58 smithi046 ceph-mon[55972]: pgmap v346: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.0 KiB/s rd, 1.3 KiB/s wr, 4 op/s 2024-09-13T13:08:59.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:59 smithi042 ceph-mon[63970]: Upgrade: Updating grafana.smithi042 2024-09-13T13:08:59.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:08:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:08:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:59 smithi046 ceph-mon[55972]: Upgrade: Updating grafana.smithi042 2024-09-13T13:08:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:08:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:00.748 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:00 smithi042 ceph-mon[63970]: Deploying daemon grafana.smithi042 on smithi042 2024-09-13T13:09:00.748 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:00 smithi042 ceph-mon[63970]: pgmap v347: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-09-13T13:09:00.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:00 smithi046 ceph-mon[55972]: Deploying daemon grafana.smithi042 on smithi042 2024-09-13T13:09:00.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:00 smithi046 ceph-mon[55972]: pgmap v347: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 204 B/s rd, 409 B/s wr, 0 op/s 2024-09-13T13:09:02.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:02 smithi042 ceph-mon[63970]: pgmap v348: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 93 B/s rd, 373 B/s wr, 0 op/s 2024-09-13T13:09:02.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:02 smithi046 ceph-mon[55972]: pgmap v348: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 93 B/s rd, 373 B/s wr, 0 op/s 2024-09-13T13:09:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:04 smithi042 ceph-mon[63970]: pgmap v349: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T13:09:04.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:04 smithi046 ceph-mon[55972]: pgmap v349: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 341 B/s wr, 0 op/s 2024-09-13T13:09:06.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:06 smithi042 ceph-mon[63970]: pgmap v350: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:06 smithi046 ceph-mon[55972]: pgmap v350: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:08.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:08 smithi042 ceph-mon[63970]: pgmap v351: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:08.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:08 smithi046 ceph-mon[55972]: pgmap v351: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:10.611 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:10 smithi042 ceph-mon[63970]: pgmap v352: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:10 smithi046 ceph-mon[55972]: pgmap v352: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:12 smithi046 ceph-mon[55972]: pgmap v353: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:12.926 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:12 smithi042 ceph-mon[63970]: pgmap v353: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:13.928 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:09:14.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:14.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:14.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:14.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:14 smithi042 ceph-mon[63970]: pgmap v354: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:14.249 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:14.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:14.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:14.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:14.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:14 smithi046 ceph-mon[55972]: pgmap v354: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:14.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:14.692 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:09:14.692 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (10m) 23s ago 22m 24.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:09:14.692 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (7m) 23s ago 22m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:09:14.692 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (7m) 25s ago 19m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 starting - - - - 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (65s) 23s ago 13m 38.0M - 19.3.0-4917-gb0201b8c 1641fe80ce72 a4b67e82e926 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (45s) 25s ago 13m 15.4M - 19.3.0-4917-gb0201b8c 1641fe80ce72 f65a07dc3c16 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (9m) 23s ago 24m 525M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (9m) 25s ago 19m 457M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (8m) 23s ago 24m 68.8M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (8m) 25s ago 19m 51.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (33s) 23s ago 13m 68.2M - 5.9 1641fe80ce72 9a9ba640d5f5 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (28s) 25s ago 13m 17.7M - 5.9 1641fe80ce72 22c9684fa79d 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (10m) 23s ago 19m 12.1M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (10m) 25s ago 19m 11.0M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (6m) 23s ago 18m 381M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (5m) 23s ago 17m 405M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:09:14.693 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (5m) 23s ago 17m 331M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:09:14.694 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (4m) 23s ago 17m 373M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:09:14.694 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (3m) 25s ago 16m 354M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:09:14.694 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (2m) 25s ago 16m 300M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:09:14.694 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (2m) 25s ago 15m 265M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 9d85dde44d01 2024-09-13T13:09:14.694 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (92s) 25s ago 15m 279M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 465a73580a0c 2024-09-13T13:09:14.694 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (9m) 23s ago 18m 57.8M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:15.136 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 14 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:09:15.137 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:09:15.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:15 smithi042 ceph-mon[63970]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:15.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:15 smithi042 ceph-mon[63970]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:15.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:15 smithi042 ceph-mon[63970]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:15.492 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:15 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/3411839019' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:15.515 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:09:15.515 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", 2024-09-13T13:09:15.515 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": true, 2024-09-13T13:09:15.515 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-13T13:09:15.515 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [ 2024-09-13T13:09:15.515 INFO:teuthology.orchestra.run.smithi042.stdout: "nfs", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "mon", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "mds", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "osd", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "crash" 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": "18/23 daemons upgraded", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "Currently upgrading grafana daemons", 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:09:15.516 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:09:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:15 smithi046 ceph-mon[55972]: from='client.34560 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:15 smithi046 ceph-mon[55972]: from='client.34564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:15 smithi046 ceph-mon[55972]: from='client.34568 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:15.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:15 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/3411839019' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:15.978 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:09:16.231 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:16 smithi042 ceph-mon[63970]: pgmap v355: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:16.232 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:16 smithi042 ceph-mon[63970]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:16.232 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:16 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/668272468' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:09:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:16 smithi046 ceph-mon[55972]: pgmap v355: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:16 smithi046 ceph-mon[55972]: from='client.34576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:16.617 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:16 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/668272468' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:09:18.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:18 smithi042 ceph-mon[63970]: pgmap v356: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:18.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:18 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:18 smithi046 ceph-mon[55972]: pgmap v356: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:18 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:20.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:20.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:19 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:20.242 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:19 smithi042 ceph-mon[63970]: pgmap v357: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:20.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:20.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:19 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:20.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:19 smithi046 ceph-mon[55972]: pgmap v357: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: pgmap v358: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:22 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T13:09:22.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: pgmap v358: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:22.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:22 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-13T13:09:23.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: Upgrade: Finalizing container_image settings 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T13:09:23.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T13:09:23.870 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.871 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: Upgrade: Complete! 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.872 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:23 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T13:09:23.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:23.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.993 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: Upgrade: Finalizing container_image settings 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-13T13:09:23.994 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-13T13:09:23.995 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: Upgrade: Complete! 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:09:23.996 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:23.997 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:23 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:24 smithi046 ceph-mon[55972]: pgmap v359: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:24.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:24 smithi042 ceph-mon[63970]: pgmap v359: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:24.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:09:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:26 smithi046 ceph-mon[55972]: pgmap v360: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:26.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:26 smithi042 ceph-mon[63970]: pgmap v360: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:28 smithi046 ceph-mon[55972]: pgmap v361: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:28.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:28 smithi042 ceph-mon[63970]: pgmap v361: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:29.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:30 smithi046 ceph-mon[55972]: pgmap v362: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:30.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:30 smithi042 ceph-mon[63970]: pgmap v362: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:32.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:32 smithi046 ceph-mon[55972]: pgmap v363: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:32.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:32 smithi042 ceph-mon[63970]: pgmap v363: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:34 smithi046 ceph-mon[55972]: pgmap v364: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:34.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:34 smithi042 ceph-mon[63970]: pgmap v364: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:36.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:36 smithi046 ceph-mon[55972]: pgmap v365: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:36.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:36 smithi042 ceph-mon[63970]: pgmap v365: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:38.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:38 smithi046 ceph-mon[55972]: pgmap v366: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:38.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:38 smithi042 ceph-mon[63970]: pgmap v366: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:40.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:40 smithi046 ceph-mon[55972]: pgmap v367: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:40.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:40 smithi042 ceph-mon[63970]: pgmap v367: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:42.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:42 smithi046 ceph-mon[55972]: pgmap v368: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:42 smithi042 ceph-mon[63970]: pgmap v368: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:44 smithi046 ceph-mon[55972]: pgmap v369: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:44.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:44 smithi042 ceph-mon[63970]: pgmap v369: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:46.663 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:46 smithi042 ceph-mon[63970]: pgmap v370: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:46.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:46 smithi046 ceph-mon[55972]: pgmap v370: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:47.073 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph orch ps' 2024-09-13T13:09:47.637 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:47 smithi042 ceph-mon[63970]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:47.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:47 smithi046 ceph-mon[55972]: from='client.34584 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:48.729 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:48 smithi042 ceph-mon[63970]: pgmap v371: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:48.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:48 smithi046 ceph-mon[55972]: pgmap v371: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:49.222 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:09:49.222 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (11m) 30s ago 23m 24.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:09:49.222 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (7m) 30s ago 23m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:09:49.222 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (7m) 59s ago 20m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:09:49.222 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (36s) 30s ago 20m 37.2M - 10.4.0 c8b91775d855 93fd235ec0ac 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (99s) 30s ago 14m 38.0M - 19.3.0-4917-gb0201b8c 1641fe80ce72 a4b67e82e926 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (80s) 59s ago 14m 15.4M - 19.3.0-4917-gb0201b8c 1641fe80ce72 f65a07dc3c16 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (10m) 30s ago 24m 533M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (9m) 59s ago 19m 457M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (9m) 30s ago 24m 67.7M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (8m) 59s ago 19m 51.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (67s) 30s ago 14m 70.7M - 5.9 1641fe80ce72 9a9ba640d5f5 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (63s) 59s ago 14m 17.7M - 5.9 1641fe80ce72 22c9684fa79d 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (11m) 30s ago 20m 12.0M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (10m) 59s ago 19m 11.0M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (7m) 30s ago 19m 382M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (6m) 30s ago 18m 405M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (5m) 30s ago 18m 331M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (4m) 30s ago 17m 374M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:09:49.223 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (4m) 59s ago 17m 354M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:09:49.224 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (3m) 59s ago 16m 300M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:09:49.224 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (2m) 59s ago 16m 265M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 9d85dde44d01 2024-09-13T13:09:49.224 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (2m) 59s ago 15m 279M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 465a73580a0c 2024-09-13T13:09:49.224 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (9m) 30s ago 19m 58.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:09:49.955 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph versions' 2024-09-13T13:09:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:50 smithi046 ceph-mon[55972]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:50.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:50 smithi046 ceph-mon[55972]: pgmap v372: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:50.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:50 smithi042 ceph-mon[63970]: from='client.34588 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:09:50.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:50 smithi042 ceph-mon[63970]: pgmap v372: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:52.098 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:09:52.098 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:09:52.098 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:09:52.098 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:52.098 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 14 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:09:52.099 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:09:52.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:52 smithi042 ceph-mon[63970]: pgmap v373: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:52.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:52 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/116784674' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:52.816 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-09-13T13:09:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:52 smithi046 ceph-mon[55972]: pgmap v373: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:52.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:52 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/116784674' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:09:54.613 INFO:teuthology.orchestra.run.smithi042.stdout:wait for servicemap items w/ changing names to refresh 2024-09-13T13:09:54.614 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:54 smithi042 ceph-mon[63970]: pgmap v374: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:54.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:54 smithi046 ceph-mon[55972]: pgmap v374: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:09:55.323 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'sleep 60' 2024-09-13T13:09:56.642 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:56 smithi042 ceph-mon[63970]: pgmap v375: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:56.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:56 smithi046 ceph-mon[55972]: pgmap v375: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:58.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:58 smithi046 ceph-mon[55972]: pgmap v376: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:58.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:58 smithi042 ceph-mon[63970]: pgmap v376: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:09:59.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:09:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:09:59.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:09:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:00.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:00 smithi046 ceph-mon[55972]: pgmap v377: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:00.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:00 smithi046 ceph-mon[55972]: overall HEALTH_OK 2024-09-13T13:10:00.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:00 smithi042 ceph-mon[63970]: pgmap v377: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:00.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:00 smithi042 ceph-mon[63970]: overall HEALTH_OK 2024-09-13T13:10:02.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:02 smithi046 ceph-mon[55972]: pgmap v378: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:02.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:02 smithi042 ceph-mon[63970]: pgmap v378: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:04.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:04 smithi046 ceph-mon[55972]: pgmap v379: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:04.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:04 smithi042 ceph-mon[63970]: pgmap v379: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:06.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:06 smithi046 ceph-mon[55972]: pgmap v380: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:06.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:06 smithi042 ceph-mon[63970]: pgmap v380: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:08.869 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:08 smithi046 ceph-mon[55972]: pgmap v381: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:08.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:08 smithi042 ceph-mon[63970]: pgmap v381: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:10.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:10 smithi046 ceph-mon[55972]: pgmap v382: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:10.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:10 smithi042 ceph-mon[63970]: pgmap v382: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:12.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:12 smithi046 ceph-mon[55972]: pgmap v383: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:12.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:12 smithi042 ceph-mon[63970]: pgmap v383: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:14 smithi046 ceph-mon[55972]: pgmap v384: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:14.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:14.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:14 smithi042 ceph-mon[63970]: pgmap v384: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:14.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:16.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:16 smithi046 ceph-mon[55972]: pgmap v385: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:16.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:16 smithi042 ceph-mon[63970]: pgmap v385: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:18.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:18 smithi046 ceph-mon[55972]: pgmap v386: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:18.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:18 smithi042 ceph-mon[63970]: pgmap v386: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:20.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:20 smithi046 ceph-mon[55972]: pgmap v387: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:20.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:20 smithi042 ceph-mon[63970]: pgmap v387: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:22.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:22 smithi046 ceph-mon[55972]: pgmap v388: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:22.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:22 smithi042 ceph-mon[63970]: pgmap v388: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: pgmap v389: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:10:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:10:24.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:10:24.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.868 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:24 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: pgmap v389: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:24.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:24 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:10:26.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:26 smithi046 ceph-mon[55972]: pgmap v390: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:26.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:26 smithi042 ceph-mon[63970]: pgmap v390: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:28.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:28 smithi046 ceph-mon[55972]: pgmap v391: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:28.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:28 smithi042 ceph-mon[63970]: pgmap v391: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:29.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:29 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:29.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:29 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:30 smithi046 ceph-mon[55972]: pgmap v392: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:30.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:30 smithi042 ceph-mon[63970]: pgmap v392: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:32.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:32 smithi046 ceph-mon[55972]: pgmap v393: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:32.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:32 smithi042 ceph-mon[63970]: pgmap v393: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:34.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:34 smithi046 ceph-mon[55972]: pgmap v394: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:34.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:34 smithi042 ceph-mon[63970]: pgmap v394: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:36.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:36 smithi042 ceph-mon[63970]: pgmap v395: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:37.118 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:36 smithi046 ceph-mon[55972]: pgmap v395: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:38.919 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:38 smithi046 ceph-mon[55972]: pgmap v396: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:38.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:38 smithi042 ceph-mon[63970]: pgmap v396: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:40.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:40 smithi042 ceph-mon[63970]: pgmap v397: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:41.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:40 smithi046 ceph-mon[55972]: pgmap v397: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:42.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:42 smithi042 ceph-mon[63970]: pgmap v398: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:43.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:42 smithi046 ceph-mon[55972]: pgmap v398: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:44 smithi042 ceph-mon[63970]: pgmap v399: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:44.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:44 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:44 smithi046 ceph-mon[55972]: pgmap v399: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:45.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:44 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:10:46.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:46 smithi042 ceph-mon[63970]: pgmap v400: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:47.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:46 smithi046 ceph-mon[55972]: pgmap v400: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:48.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:48 smithi042 ceph-mon[63970]: pgmap v401: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:49.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:48 smithi046 ceph-mon[55972]: pgmap v401: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:50.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:50 smithi042 ceph-mon[63970]: pgmap v402: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:51.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:50 smithi046 ceph-mon[55972]: pgmap v402: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:52.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:52 smithi042 ceph-mon[63970]: pgmap v403: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:53.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:52 smithi046 ceph-mon[55972]: pgmap v403: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:54.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:54 smithi042 ceph-mon[63970]: pgmap v404: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:55.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:54 smithi046 ceph-mon[55972]: pgmap v404: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:10:56.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:56 smithi042 ceph-mon[63970]: pgmap v405: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:57.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:56 smithi046 ceph-mon[55972]: pgmap v405: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:57.821 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph orch ps' 2024-09-13T13:10:58.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:58 smithi042 ceph-mon[63970]: pgmap v406: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:59.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:58 smithi046 ceph-mon[55972]: pgmap v406: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:10:59.814 INFO:teuthology.orchestra.run.smithi042.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T13:10:59.814 INFO:teuthology.orchestra.run.smithi042.stdout:alertmanager.smithi042 smithi042 *:9093,9094 running (12m) 101s ago 24m 24.5M - 0.27.0 11f11916f8cd 8e77f36aef0b 2024-09-13T13:10:59.814 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi042 smithi042 running (9m) 101s ago 24m 6899k - 19.3.0-4917-gb0201b8c 1641fe80ce72 422876236c9b 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:crash.smithi046 smithi046 running (9m) 2m ago 21m 6891k - 19.3.0-4917-gb0201b8c 1641fe80ce72 249c10641630 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:grafana.smithi042 smithi042 *:3000 running (107s) 101s ago 21m 37.2M - 10.4.0 c8b91775d855 93fd235ec0ac 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi042.hljzvq smithi042 running (2m) 101s ago 15m 38.0M - 19.3.0-4917-gb0201b8c 1641fe80ce72 a4b67e82e926 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:mds.foofs.smithi046.nqnolk smithi046 running (2m) 2m ago 15m 15.4M - 19.3.0-4917-gb0201b8c 1641fe80ce72 f65a07dc3c16 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi042.jhsfyp smithi042 *:8443,9283,8765 running (11m) 101s ago 25m 533M - 19.3.0-4917-gb0201b8c 1641fe80ce72 86c34255268f 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:mgr.smithi046.hcgddm smithi046 *:8443,9283,8765 running (10m) 2m ago 21m 457M - 19.3.0-4917-gb0201b8c 1641fe80ce72 11a8884e7b7e 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi042 smithi042 running (10m) 101s ago 26m 67.7M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 e5fd881ced8f 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:mon.smithi046 smithi046 running (9m) 2m ago 21m 51.2M 2048M 19.3.0-4917-gb0201b8c 1641fe80ce72 ff1c51b08a86 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.0.0.smithi042.fnqkqg smithi042 *:2049 running (2m) 101s ago 15m 70.7M - 5.9 1641fe80ce72 9a9ba640d5f5 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:nfs.foo.1.0.smithi046.jjnsrb smithi046 *:2049 running (2m) 2m ago 15m 17.7M - 5.9 1641fe80ce72 22c9684fa79d 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi042 smithi042 *:9100 running (12m) 101s ago 21m 12.0M - 1.7.0 72c9c2088986 fea460f57819 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:node-exporter.smithi046 smithi046 *:9100 running (11m) 2m ago 20m 11.0M - 1.7.0 72c9c2088986 1d52f6599bfa 2024-09-13T13:10:59.815 INFO:teuthology.orchestra.run.smithi042.stdout:osd.0 smithi042 running (8m) 101s ago 20m 382M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 876b70210855 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.1 smithi042 running (7m) 101s ago 19m 405M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 d1efc2076d22 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.2 smithi042 running (6m) 101s ago 19m 331M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 401fe9c119d8 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.3 smithi042 running (5m) 101s ago 18m 374M 1937M 19.3.0-4917-gb0201b8c 1641fe80ce72 fe8e512aee80 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.4 smithi046 running (5m) 2m ago 18m 354M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 fec2c43d3f4a 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.5 smithi046 running (4m) 2m ago 18m 300M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 174b6214b615 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.6 smithi046 running (3m) 2m ago 17m 265M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 9d85dde44d01 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:osd.7 smithi046 running (3m) 2m ago 17m 279M 2705M 19.3.0-4917-gb0201b8c 1641fe80ce72 465a73580a0c 2024-09-13T13:10:59.816 INFO:teuthology.orchestra.run.smithi042.stdout:prometheus.smithi042 smithi042 *:9095 running (10m) 101s ago 20m 58.3M - 2.51.0 1d3b7f56885b cc14f9ed9ab7 2024-09-13T13:10:59.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:10:59 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:11:00.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:10:59 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:11:00.398 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph orch upgrade status' 2024-09-13T13:11:00.907 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:00 smithi042 ceph-mon[63970]: pgmap v407: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:00.907 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:00 smithi042 ceph-mon[63970]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:00 smithi046 ceph-mon[55972]: pgmap v407: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:01.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:00 smithi046 ceph-mon[55972]: from='client.34596 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:02.511 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:11:02.511 INFO:teuthology.orchestra.run.smithi042.stdout: "target_image": null, 2024-09-13T13:11:02.511 INFO:teuthology.orchestra.run.smithi042.stdout: "in_progress": false, 2024-09-13T13:11:02.511 INFO:teuthology.orchestra.run.smithi042.stdout: "which": "", 2024-09-13T13:11:02.511 INFO:teuthology.orchestra.run.smithi042.stdout: "services_complete": [], 2024-09-13T13:11:02.512 INFO:teuthology.orchestra.run.smithi042.stdout: "progress": null, 2024-09-13T13:11:02.512 INFO:teuthology.orchestra.run.smithi042.stdout: "message": "", 2024-09-13T13:11:02.512 INFO:teuthology.orchestra.run.smithi042.stdout: "is_paused": false 2024-09-13T13:11:02.512 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:11:02.935 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:02 smithi042 ceph-mon[63970]: pgmap v408: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:03.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:02 smithi046 ceph-mon[55972]: pgmap v408: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:03.173 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph health detail' 2024-09-13T13:11:03.741 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:03 smithi042 ceph-mon[63970]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:04.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:03 smithi046 ceph-mon[55972]: from='client.34600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:04.742 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:04 smithi042 ceph-mon[63970]: pgmap v409: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:11:05.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:04 smithi046 ceph-mon[55972]: pgmap v409: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:11:05.314 INFO:teuthology.orchestra.run.smithi042.stdout:HEALTH_OK 2024-09-13T13:11:05.921 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:05 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/2731915835' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:11:05.948 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph versions' 2024-09-13T13:11:06.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:05 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/2731915835' entity='client.admin' cmd={"prefix": "health", "detail": "detail"} : dispatch 2024-09-13T13:11:06.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:06 smithi042 ceph-mon[63970]: pgmap v410: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:07.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:06 smithi046 ceph-mon[55972]: pgmap v410: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:08.076 INFO:teuthology.orchestra.run.smithi042.stdout:{ 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "mon": { 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "mgr": { 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "osd": { 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 8 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "mds": { 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 2 2024-09-13T13:11:08.077 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:11:08.078 INFO:teuthology.orchestra.run.smithi042.stdout: "overall": { 2024-09-13T13:11:08.078 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)": 14 2024-09-13T13:11:08.078 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:11:08.078 INFO:teuthology.orchestra.run.smithi042.stdout:} 2024-09-13T13:11:08.903 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:08 smithi042 ceph-mon[63970]: pgmap v411: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:08.903 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:08 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1351004717' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:11:08.929 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-13T13:11:09.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:08 smithi046 ceph-mon[55972]: pgmap v411: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:09.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:08 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1351004717' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:11:10.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:10 smithi042 ceph-mon[63970]: pgmap v412: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:11.065 INFO:teuthology.orchestra.run.smithi042.stdout:true 2024-09-13T13:11:11.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:10 smithi046 ceph-mon[55972]: pgmap v412: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:11.652 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -e sha1=b0201b8c79733293453e7f10a10c7fa43119222b -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-13T13:11:11.962 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:11 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1621643210' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:11:12.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:11 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1621643210' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:11:12.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:12 smithi042 ceph-mon[63970]: pgmap v413: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:13.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:12 smithi046 ceph-mon[55972]: pgmap v413: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:13.932 INFO:teuthology.orchestra.run.smithi042.stdout: "ceph version 19.3.0-4917-gb0201b8c (b0201b8c79733293453e7f10a10c7fa43119222b) squid (dev)" 2024-09-13T13:11:14.659 INFO:teuthology.task.sequential:In sequential, running task cephadm.wait_for_service... 2024-09-13T13:11:14.660 INFO:tasks.cephadm:Waiting for ceph service nfs.foo to start (timeout 300)... 2024-09-13T13:11:14.661 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- ceph orch ls -f json 2024-09-13T13:11:14.968 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:14 smithi042 ceph-mon[63970]: pgmap v414: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:11:14.969 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:14 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:11:14.969 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:14 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1912410141' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:11:15.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:14 smithi046 ceph-mon[55972]: pgmap v414: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:11:15.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:14 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-13T13:11:15.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:14 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1912410141' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-13T13:11:16.670 INFO:teuthology.orchestra.run.smithi042.stdout: 2024-09-13T13:11:16.670 INFO:teuthology.orchestra.run.smithi042.stdout:[{"placement": {"count": 1}, "service_name": "alertmanager", "service_type": "alertmanager", "status": {"created": "2024-09-13T12:45:46.197476Z", "last_refresh": "2024-09-13T13:09:18.333727Z", "ports": [9093, 9094], "running": 1, "size": 1}}, {"placement": {"host_pattern": "*"}, "service_name": "crash", "service_type": "crash", "status": {"created": "2024-09-13T12:45:40.455782Z", "last_refresh": "2024-09-13T13:08:49.406114Z", "running": 2, "size": 2}}, {"placement": {"count": 1}, "service_name": "grafana", "service_type": "grafana", "spec": {"anonymous_access": true, "protocol": "https"}, "status": {"created": "2024-09-13T12:45:43.585390Z", "last_refresh": "2024-09-13T13:09:18.333876Z", "ports": [3000], "running": 1, "size": 1}}, {"placement": {"count": 2}, "service_id": "foofs", "service_name": "mds.foofs", "service_type": "mds", "status": {"created": "2024-09-13T12:55:14.969249Z", "last_refresh": "2024-09-13T13:08:49.406694Z", "running": 2, "size": 2}}, {"placement": {"count": 2}, "service_name": "mgr", "service_type": "mgr", "status": {"created": "2024-09-13T12:45:38.212274Z", "last_refresh": "2024-09-13T13:08:49.406209Z", "running": 2, "size": 2}}, {"placement": {"count": 2, "hosts": ["smithi042:172.21.15.42=smithi042", "smithi046:172.21.15.46=smithi046"]}, "service_name": "mon", "service_type": "mon", "status": {"created": "2024-09-13T12:46:27.928075Z", "last_refresh": "2024-09-13T13:08:49.406282Z", "running": 2, "size": 2}}, {"events": ["2024-09-13T13:10:24.307529Z service:nfs.foo [INFO] \"service was created\""], "placement": {"count": 2}, "service_id": "foo", "service_name": "nfs.foo", "service_type": "nfs", "spec": {"port": 2049}, "status": {"created": "2024-09-13T12:55:35.457809Z", "last_refresh": "2024-09-13T13:08:49.406763Z", "ports": [2049], "running": 2, "size": 2}}, {"placement": {"host_pattern": "*"}, "service_name": "node-exporter", "service_type": "node-exporter", "status": {"created": "2024-09-13T12:45:44.804740Z", "last_refresh": "2024-09-13T13:08:49.406353Z", "ports": [9100], "running": 2, "size": 2}}, {"service_name": "osd", "service_type": "osd", "spec": {"filter_logic": "AND", "objectstore": "bluestore"}, "status": {"container_image_id": "1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031", "container_image_name": "quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b", "last_refresh": "2024-09-13T13:08:49.406423Z", "running": 8, "size": 8}, "unmanaged": true}, {"placement": {"count": 1}, "service_name": "prometheus", "service_type": "prometheus", "status": {"created": "2024-09-13T12:45:42.338062Z", "last_refresh": "2024-09-13T13:09:18.334015Z", "ports": [9095], "running": 1, "size": 1}}] 2024-09-13T13:11:16.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:16 smithi042 ceph-mon[63970]: pgmap v415: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:17.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:16 smithi046 ceph-mon[55972]: pgmap v415: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:17.306 INFO:tasks.cephadm:nfs.foo has 2/2 2024-09-13T13:11:17.306 DEBUG:teuthology.parallel:result is None 2024-09-13T13:11:17.306 INFO:teuthology.run_tasks:Running task vip.exec... 2024-09-13T13:11:17.317 INFO:tasks.vip:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T13:11:17.317 DEBUG:teuthology.orchestra.run.smithi042:> sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'umount /mnt/foo' 2024-09-13T13:11:17.344 INFO:teuthology.orchestra.run.smithi042.stderr:+ umount /mnt/foo 2024-09-13T13:11:17.371 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T13:11:17.382 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T13:11:17.382 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs cluster ls | grep foo' 2024-09-13T13:11:17.808 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:17 smithi042 ceph-mon[63970]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T13:11:18.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:17 smithi046 ceph-mon[55972]: from='client.34620 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T13:11:18.873 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:18 smithi042 ceph-mon[63970]: pgmap v416: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:19.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:18 smithi046 ceph-mon[55972]: pgmap v416: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:19.740 INFO:teuthology.orchestra.run.smithi042.stdout: "foo" 2024-09-13T13:11:20.330 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'ceph nfs export ls foo --detailed' 2024-09-13T13:11:20.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:20 smithi042 ceph-mon[63970]: pgmap v417: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:20.893 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:20 smithi042 ceph-mon[63970]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:20 smithi046 ceph-mon[55972]: pgmap v417: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:21.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:20 smithi046 ceph-mon[55972]: from='client.34624 -' entity='client.admin' cmd=[{"prefix": "nfs cluster ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout:[ 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout: { 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout: "access_type": "RW", 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout: "clients": [], 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout: "cluster_id": "foo", 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout: "export_id": 1, 2024-09-13T13:11:22.426 INFO:teuthology.orchestra.run.smithi042.stdout: "fsal": { 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "fs_name": "foofs", 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "name": "CEPH", 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "user_id": "nfs.foo.1" 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: }, 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "path": "/", 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "protocols": [ 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: 4 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: ], 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "pseudo": "/fake", 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "security_label": true, 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "squash": "none", 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "transports": [ 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: "TCP" 2024-09-13T13:11:22.427 INFO:teuthology.orchestra.run.smithi042.stdout: ] 2024-09-13T13:11:22.428 INFO:teuthology.orchestra.run.smithi042.stdout: } 2024-09-13T13:11:22.428 INFO:teuthology.orchestra.run.smithi042.stdout:] 2024-09-13T13:11:22.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:22 smithi042 ceph-mon[63970]: pgmap v418: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:23.077 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'rados -p .nfs --all ls -' 2024-09-13T13:11:23.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:22 smithi046 ceph-mon[55972]: pgmap v418: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-09-13T13:11:23.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:23 smithi042 ceph-mon[63970]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:24.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:23 smithi046 ceph-mon[55972]: from='client.34628 -' entity='client.admin' cmd=[{"prefix": "nfs export ls", "cluster_id": "foo", "detailed": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T13:11:24.904 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:24 smithi042 ceph-mon[63970]: pgmap v419: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:11:24.905 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:24 smithi042 ceph-mon[63970]: mgrmap e44: smithi042.jhsfyp(active, since 10m), standbys: smithi046.hcgddm 2024-09-13T13:11:24.967 INFO:teuthology.orchestra.run.smithi042.stdout:foo rec-0000000000000004:nfs.foo.1 2024-09-13T13:11:24.967 INFO:teuthology.orchestra.run.smithi042.stdout:foo grace 2024-09-13T13:11:24.967 INFO:teuthology.orchestra.run.smithi042.stdout:foo rec-0000000000000004:nfs.foo.0 2024-09-13T13:11:24.967 INFO:teuthology.orchestra.run.smithi042.stdout:foo conf-nfs.foo 2024-09-13T13:11:24.967 INFO:teuthology.orchestra.run.smithi042.stdout:foo export-1 2024-09-13T13:11:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:24 smithi046 ceph-mon[55972]: pgmap v419: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-13T13:11:25.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:24 smithi046 ceph-mon[55972]: mgrmap e44: smithi042.jhsfyp(active, since 10m), standbys: smithi046.hcgddm 2024-09-13T13:11:25.553 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T13:11:25.563 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T13:11:25.563 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 -- bash -c 'set -ex 2024-09-13T13:11:25.563 DEBUG:teuthology.orchestra.run.smithi042:> [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] 2024-09-13T13:11:25.563 DEBUG:teuthology.orchestra.run.smithi042:> ' 2024-09-13T13:11:25.868 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:25 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:11:25.869 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:25 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:11:25.869 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:25 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:11:25.869 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:25 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:11:25.869 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:25 smithi042 ceph-mon[63970]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:11:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:25 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-13T13:11:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:25 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-13T13:11:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:25 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-13T13:11:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:25 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:11:26.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:25 smithi046 ceph-mon[55972]: from='mgr.34100 172.21.15.42:0/1284885811' entity='mgr.smithi042.jhsfyp' 2024-09-13T13:11:26.961 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:26 smithi042 ceph-mon[63970]: pgmap v420: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T13:11:27.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:26 smithi046 ceph-mon[55972]: pgmap v420: 97 pgs: 97 active+clean; 249 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 op/s 2024-09-13T13:11:27.314 INFO:teuthology.orchestra.run.smithi042.stderr:++ ceph config get mgr mgr/cephadm/migration_current 2024-09-13T13:11:27.558 INFO:teuthology.orchestra.run.smithi042.stderr:+ [[ 8 -gt 2 ]] 2024-09-13T13:11:27.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:27 smithi042 ceph-mon[63970]: from='client.? 172.21.15.42:0/1948720299' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-13T13:11:28.117 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:27 smithi046 ceph-mon[55972]: from='client.? 172.21.15.42:0/1948720299' entity='client.admin' cmd={"prefix": "config get", "who": "mgr", "key": "mgr/cephadm/migration_current"} : dispatch 2024-09-13T13:11:28.211 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T13:11:28.221 INFO:tasks.cephadm:Teardown begin 2024-09-13T13:11:28.221 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T13:11:28.250 DEBUG:teuthology.orchestra.run.smithi046:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T13:11:28.285 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T13:11:28.285 DEBUG:teuthology.orchestra.run.smithi042:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T13:11:28.306 DEBUG:teuthology.orchestra.run.smithi046:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T13:11:28.343 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T13:11:28.344 INFO:tasks.cephadm.mon.smithi042:Stopping mon.smithi042... 2024-09-13T13:11:28.344 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042 2024-09-13T13:11:28.666 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:28 smithi042 systemd[1]: Stopping Ceph mon.smithi042 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:11:28.991 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[63966]: 2024-09-13T13:11:28.664+0000 7fa936278640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi042 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:11:28.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:28 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042[63966]: 2024-09-13T13:11:28.664+0000 7fa936278640 -1 mon.smithi042@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T13:11:28.992 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:28 smithi042 podman[93892]: 2024-09-13 13:11:28.848841465 +0000 UTC m=+0.301468249 container died e5fd881ced8f144de80c1ee25733aadb129cba574d4a3c95bdf7747e28700e19 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-13T13:11:29.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:29 smithi042 podman[93892]: 2024-09-13 13:11:29.23985627 +0000 UTC m=+0.692483043 container cleanup e5fd881ced8f144de80c1ee25733aadb129cba574d4a3c95bdf7747e28700e19 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD) 2024-09-13T13:11:29.491 INFO:journalctl@ceph.mon.smithi042.smithi042.stdout:Sep 13 13:11:29 smithi042 bash[93892]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi042 2024-09-13T13:11:29.631 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi042.service' 2024-09-13T13:11:29.681 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:11:29.682 INFO:tasks.cephadm.mon.smithi042:Stopped mon.smithi042 2024-09-13T13:11:29.682 INFO:tasks.cephadm.mon.smithi046:Stopping mon.smithi046... 2024-09-13T13:11:29.682 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi046 2024-09-13T13:11:30.004 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:29 smithi046 systemd[1]: Stopping Ceph mon.smithi046 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:11:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046[55968]: 2024-09-13T13:11:30.003+0000 7f1d22d39640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi046 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:11:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046[55968]: 2024-09-13T13:11:30.003+0000 7f1d22d39640 -1 mon.smithi046@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T13:11:30.367 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:30 smithi046 podman[75039]: 2024-09-13 13:11:30.207904398 +0000 UTC m=+0.323454625 container died ff1c51b08a869e0cbd4175e6613debe3c98dcec086e8854a8c862a51a3c5b19e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:11:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:30 smithi046 podman[75039]: 2024-09-13 13:11:30.669957893 +0000 UTC m=+0.785508159 container cleanup ff1c51b08a869e0cbd4175e6613debe3c98dcec086e8854a8c862a51a3c5b19e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T13:11:30.867 INFO:journalctl@ceph.mon.smithi046.smithi046.stdout:Sep 13 13:11:30 smithi046 bash[75039]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-mon-smithi046 2024-09-13T13:11:31.104 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@mon.smithi046.service' 2024-09-13T13:11:31.151 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:11:31.151 INFO:tasks.cephadm.mon.smithi046:Stopped mon.smithi046 2024-09-13T13:11:31.151 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T13:11:31.151 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.0 2024-09-13T13:11:31.486 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:31 smithi042 systemd[1]: Stopping Ceph osd.0 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:11:31.741 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:31 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:11:31.485+0000 7f7f44e37640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:11:31.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:31 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:11:31.485+0000 7f7f44e37640 -1 osd.0 169 *** Got signal Terminated *** 2024-09-13T13:11:31.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:31 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0[68762]: 2024-09-13T13:11:31.485+0000 7f7f44e37640 -1 osd.0 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:11:37.297 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:37 smithi042 podman[94027]: 2024-09-13 13:11:37.000621829 +0000 UTC m=+5.645000018 container died 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T13:11:37.631 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:37 smithi042 podman[94027]: 2024-09-13 13:11:37.40302623 +0000 UTC m=+6.047404408 container cleanup 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:11:37.631 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:37 smithi042 bash[94027]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0 2024-09-13T13:11:37.991 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:37 smithi042 podman[94057]: 2024-09-13 13:11:37.646112826 +0000 UTC m=+0.645084724 container remove 876b70210855e8fa5ec48c8a33fd940b778366598f013e02ecf6deb4cf6c47e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-13T13:11:38.300 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:38 smithi042 podman[94156]: 2024-09-13 13:11:37.909267269 +0000 UTC m=+0.017107131 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:11:38.301 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:38 smithi042 podman[94156]: 2024-09-13 13:11:38.030560084 +0000 UTC m=+0.138399930 container create b4abb7d27e80fdcc8535502cf24f03a2cf001ad4309aced50ba3576045933cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-13T13:11:38.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:38 smithi042 podman[94156]: 2024-09-13 13:11:38.300270212 +0000 UTC m=+0.408110068 container init b4abb7d27e80fdcc8535502cf24f03a2cf001ad4309aced50ba3576045933cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T13:11:38.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:38 smithi042 podman[94156]: 2024-09-13 13:11:38.304247492 +0000 UTC m=+0.412087340 container start b4abb7d27e80fdcc8535502cf24f03a2cf001ad4309aced50ba3576045933cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-13T13:11:38.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:38 smithi042 podman[94156]: 2024-09-13 13:11:38.391682122 +0000 UTC m=+0.499521979 container attach b4abb7d27e80fdcc8535502cf24f03a2cf001ad4309aced50ba3576045933cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0) 2024-09-13T13:11:38.742 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:38 smithi042 podman[94156]: 2024-09-13 13:11:38.503198932 +0000 UTC m=+0.611038796 container died b4abb7d27e80fdcc8535502cf24f03a2cf001ad4309aced50ba3576045933cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:11:39.107 INFO:journalctl@ceph.osd.0.smithi042.stdout:Sep 13 13:11:39 smithi042 podman[94180]: 2024-09-13 13:11:39.104300486 +0000 UTC m=+0.590133459 container remove b4abb7d27e80fdcc8535502cf24f03a2cf001ad4309aced50ba3576045933cf6 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-13T13:11:39.123 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T13:11:39.165 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:11:39.166 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T13:11:39.166 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T13:11:39.166 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.1 2024-09-13T13:11:39.491 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:39 smithi042 systemd[1]: Stopping Ceph osd.1 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:11:39.991 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:39 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:11:39.518+0000 7fe11c4ce640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:11:39.992 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:39 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:11:39.518+0000 7fe11c4ce640 -1 osd.1 169 *** Got signal Terminated *** 2024-09-13T13:11:39.992 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:39 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1[71300]: 2024-09-13T13:11:39.518+0000 7fe11c4ce640 -1 osd.1 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:11:45.359 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:45 smithi042 podman[94259]: 2024-09-13 13:11:45.08673865 +0000 UTC m=+5.694034428 container died d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T13:11:45.741 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:45 smithi042 podman[94259]: 2024-09-13 13:11:45.457518781 +0000 UTC m=+6.064814547 container cleanup d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:11:45.742 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:45 smithi042 bash[94259]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1 2024-09-13T13:11:46.083 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:45 smithi042 podman[94273]: 2024-09-13 13:11:45.74239408 +0000 UTC m=+0.650355065 container remove d1efc2076d2253c7365cfdc8ec61bb68ca6e615bba6b361d5b92fd1d57553bcf (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T13:11:46.388 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:46 smithi042 podman[94362]: 2024-09-13 13:11:45.98587422 +0000 UTC m=+0.016646538 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:11:46.388 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:46 smithi042 podman[94362]: 2024-09-13 13:11:46.093612897 +0000 UTC m=+0.124385204 container create 05f298458008c218d4cacfd74c8902076289045691fd0da9d566e07efa61bdb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T13:11:46.388 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:46 smithi042 podman[94362]: 2024-09-13 13:11:46.312759143 +0000 UTC m=+0.343531459 container init 05f298458008c218d4cacfd74c8902076289045691fd0da9d566e07efa61bdb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-13T13:11:46.388 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:46 smithi042 podman[94362]: 2024-09-13 13:11:46.316199621 +0000 UTC m=+0.346971934 container start 05f298458008c218d4cacfd74c8902076289045691fd0da9d566e07efa61bdb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-13T13:11:46.388 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:46 smithi042 podman[94362]: 2024-09-13 13:11:46.387711888 +0000 UTC m=+0.418484217 container attach 05f298458008c218d4cacfd74c8902076289045691fd0da9d566e07efa61bdb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:11:46.742 INFO:journalctl@ceph.osd.1.smithi042.stdout:Sep 13 13:11:46 smithi042 podman[94362]: 2024-09-13 13:11:46.513919005 +0000 UTC m=+0.544691314 container died 05f298458008c218d4cacfd74c8902076289045691fd0da9d566e07efa61bdb2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T13:11:47.129 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T13:11:47.169 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:11:47.170 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T13:11:47.170 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T13:11:47.170 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.2 2024-09-13T13:11:47.492 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:47 smithi042 systemd[1]: Stopping Ceph osd.2 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:11:47.908 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:47 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:11:47.523+0000 7f87a5fcc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:11:47.908 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:47 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:11:47.523+0000 7f87a5fcc640 -1 osd.2 169 *** Got signal Terminated *** 2024-09-13T13:11:47.908 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:47 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2[73630]: 2024-09-13T13:11:47.523+0000 7f87a5fcc640 -1 osd.2 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:11:53.209 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:52 smithi042 podman[94463]: 2024-09-13 13:11:52.899141041 +0000 UTC m=+5.501364320 container died 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-13T13:11:53.492 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:53 smithi042 podman[94463]: 2024-09-13 13:11:53.303106568 +0000 UTC m=+5.905329846 container cleanup 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2) 2024-09-13T13:11:53.492 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:53 smithi042 bash[94463]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2 2024-09-13T13:11:53.856 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:53 smithi042 podman[94479]: 2024-09-13 13:11:53.5294418 +0000 UTC m=+0.629904760 container remove 401fe9c119d8a883edec72f14a94924c7fa5117fed2f456665896c7a1b03ea4b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True) 2024-09-13T13:11:54.191 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:53 smithi042 podman[94569]: 2024-09-13 13:11:53.855321401 +0000 UTC m=+0.109608926 container create e7dda19dfd14dcd55152c0d1737fe9784e253f9f57ad6417161e80b773394593 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-13T13:11:54.192 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:53 smithi042 podman[94569]: 2024-09-13 13:11:53.762325231 +0000 UTC m=+0.016612725 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:11:54.192 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:54 smithi042 podman[94569]: 2024-09-13 13:11:54.082738019 +0000 UTC m=+0.337025511 container init e7dda19dfd14dcd55152c0d1737fe9784e253f9f57ad6417161e80b773394593 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731) 2024-09-13T13:11:54.192 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:54 smithi042 podman[94569]: 2024-09-13 13:11:54.08610731 +0000 UTC m=+0.340394796 container start e7dda19dfd14dcd55152c0d1737fe9784e253f9f57ad6417161e80b773394593 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-13T13:11:54.192 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:54 smithi042 podman[94569]: 2024-09-13 13:11:54.191416978 +0000 UTC m=+0.445704461 container attach e7dda19dfd14dcd55152c0d1737fe9784e253f9f57ad6417161e80b773394593 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T13:11:54.491 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:54 smithi042 podman[94569]: 2024-09-13 13:11:54.282933608 +0000 UTC m=+0.537221099 container died e7dda19dfd14dcd55152c0d1737fe9784e253f9f57ad6417161e80b773394593 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-13T13:11:54.873 INFO:journalctl@ceph.osd.2.smithi042.stdout:Sep 13 13:11:54 smithi042 podman[94591]: 2024-09-13 13:11:54.853423034 +0000 UTC m=+0.564518046 container remove e7dda19dfd14dcd55152c0d1737fe9784e253f9f57ad6417161e80b773394593 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS) 2024-09-13T13:11:54.879 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T13:11:54.922 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:11:54.922 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T13:11:54.922 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T13:11:54.922 DEBUG:teuthology.orchestra.run.smithi042:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.3 2024-09-13T13:11:55.242 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:11:55 smithi042 systemd[1]: Stopping Ceph osd.3 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:11:55.741 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:11:55 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:11:55.275+0000 7fecbf281640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:11:55.742 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:11:55 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:11:55.275+0000 7fecbf281640 -1 osd.3 169 *** Got signal Terminated *** 2024-09-13T13:11:55.742 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:11:55 smithi042 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3[76152]: 2024-09-13T13:11:55.275+0000 7fecbf281640 -1 osd.3 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:11:58.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:11:58 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:11:58.251+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:11:59.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:11:59 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:11:59.216+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:00.115 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:11:59 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:11:59.888+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:00.115 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:11:59 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:11:59.891+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:00.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:00 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:00.245+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:00.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:00 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:00.113+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:01.083 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:00 smithi042 podman[94673]: 2024-09-13 13:12:00.818974408 +0000 UTC m=+5.666646789 container died fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:12:01.217 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:00 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:00.933+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:01.217 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:01 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:01.093+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:01.218 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:00 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:00.919+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:01.218 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:01 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:01.216+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:01.458 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:01 smithi042 podman[94673]: 2024-09-13 13:12:01.173215295 +0000 UTC m=+6.020887666 container cleanup fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-13T13:12:01.458 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:01 smithi042 bash[94673]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3 2024-09-13T13:12:01.741 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:01 smithi042 podman[94705]: 2024-09-13 13:12:01.457807643 +0000 UTC m=+0.637982787 container remove fe8e512aee804556122736f1f4971c7c85667bc7a03f3bda05def6596cc36c28 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-13T13:12:02.086 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:01 smithi042 podman[94794]: 2024-09-13 13:12:01.70961101 +0000 UTC m=+0.017076148 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:12:02.086 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:01 smithi042 podman[94794]: 2024-09-13 13:12:01.825529254 +0000 UTC m=+0.132994431 container create e77d90df9f3ae9c75976d9710b4a1fc1cb96114df6a3e4060fb4583f9c6fc401 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-13T13:12:02.087 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:02 smithi042 podman[94794]: 2024-09-13 13:12:02.08613086 +0000 UTC m=+0.393595996 container init e77d90df9f3ae9c75976d9710b4a1fc1cb96114df6a3e4060fb4583f9c6fc401 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T13:12:02.370 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:02 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:02.178+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:02.371 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:02 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:02.057+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:02.371 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:01 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:01.931+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:02.372 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:01 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:01.940+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:02.492 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:02 smithi042 podman[94794]: 2024-09-13 13:12:02.089731161 +0000 UTC m=+0.397196303 container start e77d90df9f3ae9c75976d9710b4a1fc1cb96114df6a3e4060fb4583f9c6fc401 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T13:12:02.492 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:02 smithi042 podman[94794]: 2024-09-13 13:12:02.161187941 +0000 UTC m=+0.468653091 container attach e77d90df9f3ae9c75976d9710b4a1fc1cb96114df6a3e4060fb4583f9c6fc401 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:12:02.492 INFO:journalctl@ceph.osd.3.smithi042.stdout:Sep 13 13:12:02 smithi042 podman[94794]: 2024-09-13 13:12:02.297401809 +0000 UTC m=+0.604866945 container died e77d90df9f3ae9c75976d9710b4a1fc1cb96114df6a3e4060fb4583f9c6fc401 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:12:02.926 DEBUG:teuthology.orchestra.run.smithi042:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T13:12:02.973 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:12:02.973 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T13:12:02.974 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T13:12:02.974 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.4 2024-09-13T13:12:03.291 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:02 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:02.944+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:03.291 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:03.140+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:03.291 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:03 smithi046 systemd[1]: Stopping Ceph osd.4 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:12:03.291 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:03.053+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:03.292 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:02 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:02.931+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:03.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:03.289+0000 7f52c404b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:12:03.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:03.289+0000 7f52c404b640 -1 osd.4 169 *** Got signal Terminated *** 2024-09-13T13:12:03.617 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:03.289+0000 7f52c404b640 -1 osd.4 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:12:04.190 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:04.015+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:04.190 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:03.893+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:04.191 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:03 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:03.897+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:04.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:04.189+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:05.193 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:04.988+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:05.194 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:04.866+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:05.194 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:04 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:04.902+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:05.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:05 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:05.192+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:06.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:05 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:05.905+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:06.117 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:06.021+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:06.118 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:05 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:05.860+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:06.118 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:05 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:05.860+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:06.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:06.213+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:07.236 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:06.992+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:07.237 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:06.882+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:07.237 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:06.882+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:07.237 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:06.894+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:07.237 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:06 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:06.894+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:07.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:07 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:07.235+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:07.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:07 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:07.235+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:08.224 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:07 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:07.913+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:08.224 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:07 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:07.913+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:08.224 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:08.017+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:36.386797+0000 front 2024-09-13T13:11:36.386889+0000 (oldest deadline 2024-09-13T13:11:59.286751+0000) 2024-09-13T13:12:08.224 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4[59892]: 2024-09-13T13:12:08.017+0000 7f52c0660640 -1 osd.4 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:39.287307+0000 front 2024-09-13T13:11:39.287181+0000 (oldest deadline 2024-09-13T13:12:07.487581+0000) 2024-09-13T13:12:08.224 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:07 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:07.927+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:08.225 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:07 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:07.927+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:08.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:08.222+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:08.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:08.222+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:09.117 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:08 smithi046 podman[75204]: 2024-09-13 13:12:08.849689275 +0000 UTC m=+5.666299154 container died fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-13T13:12:09.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:08.879+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:09.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:08.879+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:09.118 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:08.970+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:09.118 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:08 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:08.970+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:09.569 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:09 smithi046 podman[75204]: 2024-09-13 13:12:09.357241501 +0000 UTC m=+6.173851371 container cleanup fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-13T13:12:09.733 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:09 smithi046 bash[75204]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4 2024-09-13T13:12:09.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:09 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:09.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:09.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:09 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:09.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:09.834 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:09 smithi046 podman[75221]: 2024-09-13 13:12:09.567377274 +0000 UTC m=+0.708033401 container remove fec2c43d3f4ab8815db5d122a819c39631eb354a7e20f9031ce3b107926cc6de (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-13T13:12:10.092 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:09 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:09.932+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:10.092 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:09 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:09.932+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:10.093 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:09 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:09.833+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:10.093 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:09 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:09.833+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:10.093 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:09 smithi046 podman[75314]: 2024-09-13 13:12:09.893174405 +0000 UTC m=+0.016646546 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:12:10.093 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:09 smithi046 podman[75314]: 2024-09-13 13:12:09.993258117 +0000 UTC m=+0.116730261 container create a3ca1aa94080c0cf2cc68e09b8dc710940ea6b15b9678726463b01e71ec873d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:12:10.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:10 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:10.231+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:10.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:10 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:10.231+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:10.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:10 smithi046 podman[75314]: 2024-09-13 13:12:10.229352917 +0000 UTC m=+0.352825059 container init a3ca1aa94080c0cf2cc68e09b8dc710940ea6b15b9678726463b01e71ec873d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:12:10.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:10 smithi046 podman[75314]: 2024-09-13 13:12:10.232860475 +0000 UTC m=+0.356332617 container start a3ca1aa94080c0cf2cc68e09b8dc710940ea6b15b9678726463b01e71ec873d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:12:10.367 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:10 smithi046 podman[75314]: 2024-09-13 13:12:10.329276433 +0000 UTC m=+0.452748575 container attach a3ca1aa94080c0cf2cc68e09b8dc710940ea6b15b9678726463b01e71ec873d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-13T13:12:10.687 INFO:journalctl@ceph.osd.4.smithi046.stdout:Sep 13 13:12:10 smithi046 podman[75314]: 2024-09-13 13:12:10.424333755 +0000 UTC m=+0.547805898 container died a3ca1aa94080c0cf2cc68e09b8dc710940ea6b15b9678726463b01e71ec873d4 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-13T13:12:10.939 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:10 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:10.827+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:10.939 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:10 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:10.827+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:11.201 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:10 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:10.938+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:11.201 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:10 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:10.938+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:11.217 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T13:12:11.261 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:12:11.261 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T13:12:11.262 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T13:12:11.262 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.5 2024-09-13T13:12:11.458 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:11 smithi046 systemd[1]: Stopping Ceph osd.5 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:12:11.458 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:11.239+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:11.459 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:11.239+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:11.828 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:11.612+0000 7fc438491640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:12:11.828 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:11.612+0000 7fc438491640 -1 osd.5 169 *** Got signal Terminated *** 2024-09-13T13:12:11.829 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:11.612+0000 7fc438491640 -1 osd.5 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:12:12.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:11.975+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:12.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:11.975+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:12.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:11.827+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:12.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:11 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:11.827+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:12.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:12.281+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:12.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:12.281+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:13.261 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:12.952+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:13.261 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:12.952+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:13.261 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:12.874+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:13.262 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:12 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:12.874+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:13.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:13.260+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:13.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:13.260+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:14.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:13.949+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:14.117 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:13.949+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:14.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:13.846+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:14.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:13.846+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:14.118 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:13 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:13.846+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:14.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:14.278+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:14.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:14.278+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:15.321 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:14.989+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:15.321 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:14.989+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:15.321 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:14.989+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.450666+0000 front 2024-09-13T13:11:51.450577+0000 (oldest deadline 2024-09-13T13:12:14.950631+0000) 2024-09-13T13:12:15.321 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:14.894+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:15.321 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:14.894+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:15.321 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:14 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:14.894+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:15.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:15.319+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:15.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:15.319+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:15.618 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:15.319+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:16.313 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:15.973+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:35.247798+0000 front 2024-09-13T13:11:35.248006+0000 (oldest deadline 2024-09-13T13:11:59.347727+0000) 2024-09-13T13:12:16.313 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:15.973+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:44.048791+0000 front 2024-09-13T13:11:44.048728+0000 (oldest deadline 2024-09-13T13:12:05.148607+0000) 2024-09-13T13:12:16.313 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5[61933]: 2024-09-13T13:12:15.973+0000 7fc434aa6640 -1 osd.5 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.450666+0000 front 2024-09-13T13:11:51.450577+0000 (oldest deadline 2024-09-13T13:12:14.950631+0000) 2024-09-13T13:12:16.313 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:15.913+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:16.313 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:15.913+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:16.313 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:15 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:15.913+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:16.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:16 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:16.312+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:16.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:16 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:16.312+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:16.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:16 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:16.312+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:17.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:16 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:16.865+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:17.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:16 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:16.865+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:17.117 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:16 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:16.865+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:17.587 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:17 smithi046 podman[75418]: 2024-09-13 13:12:17.199615778 +0000 UTC m=+5.709688940 container died 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-13T13:12:17.588 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:17 smithi046 podman[75418]: 2024-09-13 13:12:17.587299632 +0000 UTC m=+6.097372793 container cleanup 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731) 2024-09-13T13:12:17.588 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:17.299+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:17.588 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:17.299+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:17.588 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:17.299+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:17.867 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:17 smithi046 bash[75418]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5 2024-09-13T13:12:18.256 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:17 smithi046 podman[75461]: 2024-09-13 13:12:17.938994468 +0000 UTC m=+0.738299271 container remove 174b6214b6159ce98c00539b1164a85abbcad4bff0b0c3cfd24644e249d05dc2 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:12:18.257 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:17.902+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:18.257 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:17.902+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:18.257 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:17 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:17.902+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:18.551 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:18 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:18.278+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:18.551 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:18 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:18.278+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:18.551 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:18 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:18.278+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:18.551 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:18 smithi046 podman[75548]: 2024-09-13 13:12:18.256337162 +0000 UTC m=+0.105964996 container create e241324468e6fe9530b06d32e8afec76d9876a154559549e027fd309c9b7938e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-13T13:12:18.551 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:18 smithi046 podman[75548]: 2024-09-13 13:12:18.166494593 +0000 UTC m=+0.016122436 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:12:18.551 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:18 smithi046 podman[75548]: 2024-09-13 13:12:18.49235984 +0000 UTC m=+0.341987676 container init e241324468e6fe9530b06d32e8afec76d9876a154559549e027fd309c9b7938e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-13T13:12:18.552 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:18 smithi046 podman[75548]: 2024-09-13 13:12:18.495691156 +0000 UTC m=+0.345318992 container start e241324468e6fe9530b06d32e8afec76d9876a154559549e027fd309c9b7938e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-13T13:12:18.552 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:18 smithi046 podman[75548]: 2024-09-13 13:12:18.550690436 +0000 UTC m=+0.400318272 container attach e241324468e6fe9530b06d32e8afec76d9876a154559549e027fd309c9b7938e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-13T13:12:18.867 INFO:journalctl@ceph.osd.5.smithi046.stdout:Sep 13 13:12:18 smithi046 podman[75548]: 2024-09-13 13:12:18.69341552 +0000 UTC m=+0.543043354 container died e241324468e6fe9530b06d32e8afec76d9876a154559549e027fd309c9b7938e (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8) 2024-09-13T13:12:19.221 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:18 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:18.926+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:19.221 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:18 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:18.926+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:19.221 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:18 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:18.926+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:19.243 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T13:12:19.287 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:12:19.287 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T13:12:19.287 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T13:12:19.287 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.6 2024-09-13T13:12:19.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 systemd[1]: Stopping Ceph osd.6 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:12:19.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:19.246+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:19.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:19.246+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:19.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:19.246+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:19.925 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:19.615+0000 7feed1bfd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:12:19.925 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:19.615+0000 7feed1bfd640 -1 osd.6 169 *** Got signal Terminated *** 2024-09-13T13:12:19.925 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:19.615+0000 7feed1bfd640 -1 osd.6 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:12:19.925 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:19.924+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:19.925 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:19.924+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:19.925 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:19 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:19.924+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:20.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:20 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:20.216+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:20.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:20 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:20.216+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:20.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:20 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:20.216+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:21.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:20 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:20.929+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:21.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:20 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:20.929+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:21.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:20 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:20.929+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:21.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:21.175+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:21.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:21.175+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:21.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:21.175+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:22.203 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:21.973+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:22.204 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:21.974+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:22.204 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:21.974+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:22.204 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:21 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:21.974+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.646489+0000 front 2024-09-13T13:11:57.646472+0000 (oldest deadline 2024-09-13T13:12:21.146437+0000) 2024-09-13T13:12:22.515 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:22.203+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:22.515 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:22.203+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:22.515 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:22.203+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:23.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:22.987+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:23.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:22.987+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:23.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:22.987+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:23.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:22 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:22.987+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.646489+0000 front 2024-09-13T13:11:57.646472+0000 (oldest deadline 2024-09-13T13:12:21.146437+0000) 2024-09-13T13:12:23.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:23.223+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:23.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:23.223+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:23.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:23.223+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:24.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:23.984+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:32.142947+0000 front 2024-09-13T13:11:32.142917+0000 (oldest deadline 2024-09-13T13:11:59.743177+0000) 2024-09-13T13:12:24.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:23.984+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.644199+0000 front 2024-09-13T13:11:42.644072+0000 (oldest deadline 2024-09-13T13:12:06.143911+0000) 2024-09-13T13:12:24.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:23.984+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:51.945269+0000 front 2024-09-13T13:11:51.945204+0000 (oldest deadline 2024-09-13T13:12:13.645176+0000) 2024-09-13T13:12:24.367 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:23 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6[63973]: 2024-09-13T13:12:23.984+0000 7feece212640 -1 osd.6 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.646489+0000 front 2024-09-13T13:11:57.646472+0000 (oldest deadline 2024-09-13T13:12:21.146437+0000) 2024-09-13T13:12:24.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:24.194+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:24.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:24.194+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:24.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:24.194+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:24.368 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:24 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:24.194+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:25.291 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:25 smithi046 podman[75651]: 2024-09-13 13:12:25.014965732 +0000 UTC m=+5.499229837 container died 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-09-13T13:12:25.291 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:25 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:25.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:25.291 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:25 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:25.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:25.291 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:25 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:25.202+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:25.291 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:25 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:25.202+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:25.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:25 smithi046 podman[75651]: 2024-09-13 13:12:25.388914201 +0000 UTC m=+5.873178303 container cleanup 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T13:12:25.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:25 smithi046 bash[75651]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6 2024-09-13T13:12:26.020 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:25 smithi046 podman[75668]: 2024-09-13 13:12:25.692548563 +0000 UTC m=+0.667203665 container remove 9d85dde44d01fed364cc6c45a50e3a672a6131a1c4d6e5093f4249e1bca46011 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T13:12:26.287 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:26 smithi046 podman[75760]: 2024-09-13 13:12:25.922752896 +0000 UTC m=+0.017613193 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:12:26.288 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:26 smithi046 podman[75760]: 2024-09-13 13:12:26.043414911 +0000 UTC m=+0.138275249 container create 2dddcac223c4efaee6b4cf4f604d31507574c30e3446f3f0c2a5065102ea4e3b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T13:12:26.288 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:26 smithi046 podman[75760]: 2024-09-13 13:12:26.287306697 +0000 UTC m=+0.382166996 container init 2dddcac223c4efaee6b4cf4f604d31507574c30e3446f3f0c2a5065102ea4e3b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=) 2024-09-13T13:12:26.288 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:26 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:26.198+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:26.288 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:26 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:26.198+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:26.288 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:26 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:26.198+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:26.288 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:26 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:26.198+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:26.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:26 smithi046 podman[75760]: 2024-09-13 13:12:26.290715524 +0000 UTC m=+0.385575821 container start 2dddcac223c4efaee6b4cf4f604d31507574c30e3446f3f0c2a5065102ea4e3b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-13T13:12:26.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:26 smithi046 podman[75760]: 2024-09-13 13:12:26.370798271 +0000 UTC m=+0.465658568 container attach 2dddcac223c4efaee6b4cf4f604d31507574c30e3446f3f0c2a5065102ea4e3b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux ) 2024-09-13T13:12:26.617 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:26 smithi046 podman[75760]: 2024-09-13 13:12:26.489999517 +0000 UTC m=+0.584859823 container died 2dddcac223c4efaee6b4cf4f604d31507574c30e3446f3f0c2a5065102ea4e3b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:12:27.074 INFO:journalctl@ceph.osd.6.smithi046.stdout:Sep 13 13:12:27 smithi046 podman[75783]: 2024-09-13 13:12:27.058021747 +0000 UTC m=+0.560899236 container remove 2dddcac223c4efaee6b4cf4f604d31507574c30e3446f3f0c2a5065102ea4e3b (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-6-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-13T13:12:27.080 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T13:12:27.124 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:12:27.125 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T13:12:27.125 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T13:12:27.125 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.7 2024-09-13T13:12:27.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.167+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:27.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.167+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:27.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.167+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:27.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.167+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:27.367 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 systemd[1]: Stopping Ceph osd.7 for e7c8d5a8-71cd-11ef-bceb-c7b262605968... 2024-09-13T13:12:27.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.460+0000 7f5d510a6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-13T13:12:27.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.460+0000 7f5d510a6640 -1 osd.7 169 *** Got signal Terminated *** 2024-09-13T13:12:27.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:27 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:27.460+0000 7f5d510a6640 -1 osd.7 169 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T13:12:28.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:28 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:28.158+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:28.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:28 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:28.158+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:28.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:28 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:28.158+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:28.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:28 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:28.158+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:29.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:29 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:29.205+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:29.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:29 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:29.205+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:29.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:29 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:29.205+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:29.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:29 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:29.205+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:30.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:30.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:30.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:30.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:30.618 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:30.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:30.618 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:30.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:30.618 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:30 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:30.252+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.46:6804 osd.4 since back 2024-09-13T13:12:07.304370+0000 front 2024-09-13T13:12:07.304435+0000 (oldest deadline 2024-09-13T13:12:29.604485+0000) 2024-09-13T13:12:31.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:31 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:31.240+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:31.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:31 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:31.240+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:31.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:31 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:31.240+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:31.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:31 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:31.240+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:31.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:31 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:31.240+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.46:6804 osd.4 since back 2024-09-13T13:12:07.304370+0000 front 2024-09-13T13:12:07.304435+0000 (oldest deadline 2024-09-13T13:12:29.604485+0000) 2024-09-13T13:12:32.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:32 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:32.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6806 osd.0 since back 2024-09-13T13:11:34.200038+0000 front 2024-09-13T13:11:34.199913+0000 (oldest deadline 2024-09-13T13:11:57.699830+0000) 2024-09-13T13:12:32.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:32 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:32.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6814 osd.1 since back 2024-09-13T13:11:42.301409+0000 front 2024-09-13T13:11:42.301338+0000 (oldest deadline 2024-09-13T13:12:06.401252+0000) 2024-09-13T13:12:32.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:32 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:32.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6822 osd.2 since back 2024-09-13T13:11:49.703061+0000 front 2024-09-13T13:11:49.703008+0000 (oldest deadline 2024-09-13T13:12:14.402838+0000) 2024-09-13T13:12:32.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:32 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:32.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.42:6830 osd.3 since back 2024-09-13T13:11:57.903600+0000 front 2024-09-13T13:11:57.903669+0000 (oldest deadline 2024-09-13T13:12:23.803527+0000) 2024-09-13T13:12:32.618 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:32 smithi046 ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7[66033]: 2024-09-13T13:12:32.201+0000 7f5d4d6bb640 -1 osd.7 169 heartbeat_check: no reply from 172.21.15.46:6804 osd.4 since back 2024-09-13T13:12:07.304370+0000 front 2024-09-13T13:12:07.304435+0000 (oldest deadline 2024-09-13T13:12:29.604485+0000) 2024-09-13T13:12:33.220 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:32 smithi046 podman[75864]: 2024-09-13 13:12:32.96366708 +0000 UTC m=+5.605931297 container died 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-09-13T13:12:33.562 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:33 smithi046 podman[75864]: 2024-09-13 13:12:33.385914228 +0000 UTC m=+6.028178436 container cleanup 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:12:33.563 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:33 smithi046 bash[75864]: ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7 2024-09-13T13:12:33.867 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:33 smithi046 podman[75892]: 2024-09-13 13:12:33.562201857 +0000 UTC m=+0.597182531 container remove 465a73580a0ce8cfcc34dda8aa6a1b2b06db71c5a01f14a5f85116c9616a95b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-13T13:12:34.274 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:34 smithi046 podman[75983]: 2024-09-13 13:12:33.911134261 +0000 UTC m=+0.016391497 image pull 1641fe80ce7289a76d112fafe763b98dbdcc395dba398d6702278c8c710a2031 quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b 2024-09-13T13:12:34.274 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:34 smithi046 podman[75983]: 2024-09-13 13:12:34.01284957 +0000 UTC m=+0.118106793 container create c5d93ff7108d8c8de9158123a52b49007293ea8e94372a64aa0cd38e938cf611 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-13T13:12:34.274 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:34 smithi046 podman[75983]: 2024-09-13 13:12:34.215857054 +0000 UTC m=+0.321114286 container init c5d93ff7108d8c8de9158123a52b49007293ea8e94372a64aa0cd38e938cf611 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-13T13:12:34.274 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:34 smithi046 podman[75983]: 2024-09-13 13:12:34.219251003 +0000 UTC m=+0.324508227 container start c5d93ff7108d8c8de9158123a52b49007293ea8e94372a64aa0cd38e938cf611 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-13T13:12:34.275 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:34 smithi046 podman[75983]: 2024-09-13 13:12:34.274078376 +0000 UTC m=+0.379335612 container attach c5d93ff7108d8c8de9158123a52b49007293ea8e94372a64aa0cd38e938cf611 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-09-13T13:12:34.617 INFO:journalctl@ceph.osd.7.smithi046.stdout:Sep 13 13:12:34 smithi046 podman[75983]: 2024-09-13 13:12:34.411432689 +0000 UTC m=+0.516689915 container died c5d93ff7108d8c8de9158123a52b49007293ea8e94372a64aa0cd38e938cf611 (image=quay.ceph.io/ceph-ci/ceph@sha256:5ab18e92f956ea4280103e54eef24f76e1a43e77fc4dc78cee3cb12c7a94538b, name=ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968-osd-7-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=wip-guits-main-2024-09-13-0840-b0201b8, GIT_BRANCH=HEAD) 2024-09-13T13:12:35.008 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-e7c8d5a8-71cd-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T13:12:35.049 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T13:12:35.050 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T13:12:35.050 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T13:15:33.409 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T13:15:54.337 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T13:15:54.374 DEBUG:teuthology.orchestra.run.smithi046:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T13:15:54.402 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T13:15:54.404 DEBUG:teuthology.misc:Transferring archived files from smithi042:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574/remote/smithi042/crash 2024-09-13T13:15:54.405 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/crash -- . 2024-09-13T13:15:54.454 INFO:teuthology.orchestra.run.smithi042.stderr:tar: /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T13:15:54.454 INFO:teuthology.orchestra.run.smithi042.stderr:tar: Error is not recoverable: exiting now 2024-09-13T13:15:54.456 DEBUG:teuthology.misc:Transferring archived files from smithi046:/var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574/remote/smithi046/crash 2024-09-13T13:15:54.457 DEBUG:teuthology.orchestra.run.smithi046:> sudo tar c -f - -C /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/crash -- . 2024-09-13T13:15:54.500 INFO:teuthology.orchestra.run.smithi046.stderr:tar: /var/lib/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T13:15:54.500 INFO:teuthology.orchestra.run.smithi046.stderr:tar: Error is not recoverable: exiting now 2024-09-13T13:15:54.501 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T13:15:54.501 DEBUG:teuthology.orchestra.run.smithi042:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v CEPHADM_REFRESH_FAILED | head -n 1 2024-09-13T13:15:54.533 INFO:tasks.cephadm:Compressing logs... 2024-09-13T13:15:54.533 DEBUG:teuthology.orchestra.run.smithi042:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T13:15:54.576 DEBUG:teuthology.orchestra.run.smithi046:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-13T13:15:54.602 INFO:teuthology.orchestra.run.smithi046.stderr:find: ‘/var/log/rbd-target-api’gzip: No such file or directory -5 2024-09-13T13:15:54.603 INFO:teuthology.orchestra.run.smithi046.stderr: --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mon.smithi046.log 2024-09-13T13:15:54.603 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-13T13:15:54.603 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mon.smithi046.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T13:15:54.603 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-13T13:15:54.603 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T13:15:54.603 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi046.nqnolk.log 2024-09-13T13:15:54.608 INFO:teuthology.orchestra.run.smithi042.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-13T13:15:54.609 INFO:teuthology.orchestra.run.smithi042.stderr:: No such file or directory 2024-09-13T13:15:54.609 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-13T13:15:54.609 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mgr.smithi042.jhsfyp.log 2024-09-13T13:15:54.610 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-13T13:15:54.611 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mgr.smithi042.jhsfyp.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-13T13:15:54.611 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T13:15:54.611 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-volume.log: /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi042.hljzvq.log 2024-09-13T13:15:54.611 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-13T13:15:54.611 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi042.hljzvq.log: 85.8% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.4.log: /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi046.nqnolk.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.cephadm.log: 87.6% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi046.nqnolk.log.gz 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mgr.smithi046.hcgddm.log 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr: 85.7% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.cephadm.log.gz/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mgr.smithi046.hcgddm.log: 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.5.log: 91.6% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T13:15:54.614 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T13:15:54.615 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log: 87.1% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T13:15:54.615 INFO:teuthology.orchestra.run.smithi042.stderr: 91.5% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-13T13:15:54.617 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/cephadm.log: 91.2% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T13:15:54.618 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log 2024-09-13T13:15:54.619 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mon.smithi042.log 2024-09-13T13:15:54.619 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-13T13:15:54.620 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mon.smithi042.log: 91.2% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-13T13:15:54.622 INFO:teuthology.orchestra.run.smithi046.stderr: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T13:15:54.624 INFO:teuthology.orchestra.run.smithi042.stderr: 87.2% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-13T13:15:54.631 INFO:teuthology.orchestra.run.smithi042.stderr: 91.4% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mds.foofs.smithi042.hljzvq.log.gz 2024-09-13T13:15:54.632 INFO:teuthology.orchestra.run.smithi042.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-13T13:15:54.673 INFO:teuthology.orchestra.run.smithi042.stderr:/var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 91.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-13T13:15:54.675 INFO:teuthology.orchestra.run.smithi046.stderr: 89.9% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mgr.smithi046.hcgddm.log.gz 2024-09-13T13:15:55.006 INFO:teuthology.orchestra.run.smithi042.stderr: 88.8% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mgr.smithi042.jhsfyp.log.gz 2024-09-13T13:15:55.092 INFO:teuthology.orchestra.run.smithi046.stderr: 92.6% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mon.smithi046.log.gz 2024-09-13T13:15:56.073 INFO:teuthology.orchestra.run.smithi042.stderr: 91.7% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-mon.smithi042.log.gz 2024-09-13T13:16:05.539 INFO:teuthology.orchestra.run.smithi042.stderr: 94.5% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-13T13:16:08.441 INFO:teuthology.orchestra.run.smithi046.stderr: 93.9% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-13T13:16:09.567 INFO:teuthology.orchestra.run.smithi046.stderr: 94.2% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-13T13:16:10.401 INFO:teuthology.orchestra.run.smithi042.stderr: 94.1% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-13T13:16:12.031 INFO:teuthology.orchestra.run.smithi042.stderr: 94.1% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-13T13:16:12.392 INFO:teuthology.orchestra.run.smithi042.stderr: 94.2% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-13T13:16:12.394 INFO:teuthology.orchestra.run.smithi042.stderr: 2024-09-13T13:16:12.394 INFO:teuthology.orchestra.run.smithi042.stderr:real 0m17.803s 2024-09-13T13:16:12.394 INFO:teuthology.orchestra.run.smithi042.stderr:user 1m1.470s 2024-09-13T13:16:12.394 INFO:teuthology.orchestra.run.smithi042.stderr:sys 0m2.292s 2024-09-13T13:16:12.901 INFO:teuthology.orchestra.run.smithi046.stderr: 94.2% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-13T13:16:18.674 INFO:teuthology.orchestra.run.smithi046.stderr: 94.1% -- replaced with /var/log/ceph/e7c8d5a8-71cd-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-13T13:16:18.675 INFO:teuthology.orchestra.run.smithi046.stderr: 2024-09-13T13:16:18.675 INFO:teuthology.orchestra.run.smithi046.stderr:real 0m24.084s 2024-09-13T13:16:18.675 INFO:teuthology.orchestra.run.smithi046.stderr:user 1m9.017s 2024-09-13T13:16:18.675 INFO:teuthology.orchestra.run.smithi046.stderr:sys 0m2.526s 2024-09-13T13:16:18.676 INFO:tasks.cephadm:Archiving logs... 2024-09-13T13:16:18.677 DEBUG:teuthology.misc:Transferring archived files from smithi042:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574/remote/smithi042/log 2024-09-13T13:16:18.678 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T13:16:27.152 DEBUG:teuthology.misc:Transferring archived files from smithi046:/var/log/ceph to /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574/remote/smithi046/log 2024-09-13T13:16:27.153 DEBUG:teuthology.orchestra.run.smithi046:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T13:16:36.824 INFO:tasks.cephadm:Removing cluster... 2024-09-13T13:16:36.824 DEBUG:teuthology.orchestra.run.smithi042:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 --force 2024-09-13T13:16:37.192 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid e7c8d5a8-71cd-11ef-bceb-c7b262605968 --force 2024-09-13T13:16:37.578 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T13:16:37.578 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T13:16:37.594 DEBUG:teuthology.orchestra.run.smithi046:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T13:16:37.611 INFO:tasks.cephadm:Teardown complete 2024-09-13T13:16:37.612 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T13:16:37.622 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T13:16:37.622 DEBUG:teuthology.orchestra.run.smithi042:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T13:16:37.637 DEBUG:teuthology.orchestra.run.smithi046:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T13:16:37.650 INFO:teuthology.orchestra.run.smithi042.stderr:bash: line 1: ntpq: command not found 2024-09-13T13:16:37.655 INFO:teuthology.orchestra.run.smithi042.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T13:16:37.655 INFO:teuthology.orchestra.run.smithi042.stdout:=============================================================================== 2024-09-13T13:16:37.655 INFO:teuthology.orchestra.run.smithi042.stdout:^* hv01.front.sepia.ceph.com 3 6 377 52 -85us[ -41us] +/- 51ms 2024-09-13T13:16:37.655 INFO:teuthology.orchestra.run.smithi042.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 50 -68us[ -68us] +/- 63ms 2024-09-13T13:16:37.655 INFO:teuthology.orchestra.run.smithi042.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 +263us[ +263us] +/- 66ms 2024-09-13T13:16:37.655 INFO:teuthology.orchestra.run.smithi042.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T13:16:37.666 INFO:teuthology.orchestra.run.smithi046.stderr:bash: line 1: ntpq: command not found 2024-09-13T13:16:37.670 INFO:teuthology.orchestra.run.smithi046.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-13T13:16:37.670 INFO:teuthology.orchestra.run.smithi046.stdout:=============================================================================== 2024-09-13T13:16:37.671 INFO:teuthology.orchestra.run.smithi046.stdout:^* hv01.front.sepia.ceph.com 3 6 377 53 -59us[ -20us] +/- 51ms 2024-09-13T13:16:37.671 INFO:teuthology.orchestra.run.smithi046.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 52 -83us[ -83us] +/- 63ms 2024-09-13T13:16:37.671 INFO:teuthology.orchestra.run.smithi046.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 51 +260us[ +260us] +/- 66ms 2024-09-13T13:16:37.671 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-13T13:16:37.671 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T13:16:37.681 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T13:16:37.681 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T13:16:37.703 DEBUG:teuthology.orchestra.run.smithi042:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T13:16:37.783 DEBUG:teuthology.orchestra.run.smithi046:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-13T13:16:37.860 DEBUG:teuthology.orchestra.run.smithi042:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T13:16:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:16:37.890 DEBUG:teuthology.orchestra.run.smithi046:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-13T13:16:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:16:37.920 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T13:16:37.929 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T13:16:37.947 INFO:teuthology.task.internal:Duration was 2166.786559 seconds 2024-09-13T13:16:37.948 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T13:16:37.970 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T13:16:37.970 DEBUG:teuthology.orchestra.run.smithi042:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T13:16:37.973 DEBUG:teuthology.orchestra.run.smithi046:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T13:16:38.011 INFO:teuthology.orchestra.run.smithi042.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T13:16:38.013 INFO:teuthology.orchestra.run.smithi046.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-13T13:16:38.533 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T13:16:38.534 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi042.front.sepia.ceph.com 2024-09-13T13:16:38.534 DEBUG:teuthology.orchestra.run.smithi042:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T13:16:38.597 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi046.front.sepia.ceph.com 2024-09-13T13:16:38.597 DEBUG:teuthology.orchestra.run.smithi046:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T13:16:38.622 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T13:16:38.622 DEBUG:teuthology.orchestra.run.smithi042:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T13:16:38.639 DEBUG:teuthology.orchestra.run.smithi046:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T13:16:38.690 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T13:16:38.690 DEBUG:teuthology.orchestra.run.smithi042:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T13:16:38.708 DEBUG:teuthology.orchestra.run.smithi046:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T13:16:53.266 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T13:16:53.276 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T13:16:53.277 DEBUG:teuthology.orchestra.run.smithi042:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T13:16:53.303 DEBUG:teuthology.orchestra.run.smithi046:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T13:16:53.335 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T13:16:53.345 DEBUG:teuthology.orchestra.run.smithi042:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T13:16:53.348 DEBUG:teuthology.orchestra.run.smithi046:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T13:16:53.369 INFO:teuthology.orchestra.run.smithi042.stdout:kernel.core_pattern = core 2024-09-13T13:16:53.403 INFO:teuthology.orchestra.run.smithi046.stdout:kernel.core_pattern = core 2024-09-13T13:16:53.429 DEBUG:teuthology.orchestra.run.smithi042:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T13:16:53.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:16:53.454 DEBUG:teuthology.orchestra.run.smithi046:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T13:16:53.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T13:16:53.486 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T13:16:53.497 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T13:16:53.498 DEBUG:teuthology.misc:Transferring archived files from smithi042:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574/remote/smithi042 2024-09-13T13:16:53.498 DEBUG:teuthology.orchestra.run.smithi042:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T13:16:53.560 DEBUG:teuthology.misc:Transferring archived files from smithi046:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/gabrioux-2024-09-13_12:30:34-orch:cephadm-wip-guits-main-2024-09-13-0840-distro-default-smithi/7903574/remote/smithi046 2024-09-13T13:16:53.560 DEBUG:teuthology.orchestra.run.smithi046:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T13:16:53.605 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T13:16:53.605 DEBUG:teuthology.orchestra.run.smithi042:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T13:16:53.608 DEBUG:teuthology.orchestra.run.smithi046:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T13:16:53.655 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T13:16:53.667 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T13:16:53.667 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T13:16:53.696 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T13:16:53.696 DEBUG:teuthology.orchestra.run.smithi042:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T13:16:53.699 DEBUG:teuthology.orchestra.run.smithi046:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T13:16:53.712 INFO:teuthology.orchestra.run.smithi042.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 13:16 /home/ubuntu/cephtest 2024-09-13T13:16:53.716 INFO:teuthology.orchestra.run.smithi046.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 13:16 /home/ubuntu/cephtest 2024-09-13T13:16:53.717 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T13:16:53.728 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T13:16:53.763 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 2166.7865586280823 owner: scheduled_gabrioux@teuthology success: true 2024-09-13T13:16:53.763 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T13:16:53.839 INFO:teuthology.run:pass